2024-12-03 05:14:41,043 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 05:14:41,058 main DEBUG Took 0.013514 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 05:14:41,059 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 05:14:41,059 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 05:14:41,061 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 05:14:41,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,073 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 05:14:41,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,094 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,095 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,097 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,098 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,099 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,100 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,104 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,105 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:14:41,106 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,106 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 05:14:41,108 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:14:41,110 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 05:14:41,116 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 05:14:41,117 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 05:14:41,118 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 05:14:41,119 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 05:14:41,129 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 05:14:41,133 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 05:14:41,136 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 05:14:41,138 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 05:14:41,139 main DEBUG createAppenders(={Console}) 2024-12-03 05:14:41,139 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 05:14:41,140 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 05:14:41,140 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 05:14:41,141 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 05:14:41,141 main DEBUG OutputStream closed 2024-12-03 05:14:41,141 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 05:14:41,141 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 05:14:41,142 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 05:14:41,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 05:14:41,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 05:14:41,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 05:14:41,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 05:14:41,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 05:14:41,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 05:14:41,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 05:14:41,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 05:14:41,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 05:14:41,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 05:14:41,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 05:14:41,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 05:14:41,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 05:14:41,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 05:14:41,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 05:14:41,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 05:14:41,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 05:14:41,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 05:14:41,249 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 05:14:41,250 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 05:14:41,250 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 05:14:41,252 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T05:14:41,605 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d 2024-12-03 05:14:41,608 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 05:14:41,608 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T05:14:41,617 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T05:14:41,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T05:14:41,662 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87, deleteOnExit=true 2024-12-03T05:14:41,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T05:14:41,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/test.cache.data in system properties and HBase conf 2024-12-03T05:14:41,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T05:14:41,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.log.dir in system properties and HBase conf 2024-12-03T05:14:41,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T05:14:41,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T05:14:41,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T05:14:41,773 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T05:14:41,871 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T05:14:41,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T05:14:41,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T05:14:41,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T05:14:41,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T05:14:41,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T05:14:41,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T05:14:41,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T05:14:41,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T05:14:41,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T05:14:41,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/nfs.dump.dir in system properties and HBase conf 2024-12-03T05:14:41,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/java.io.tmpdir in system properties and HBase conf 2024-12-03T05:14:41,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T05:14:41,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T05:14:41,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T05:14:43,061 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T05:14:43,140 INFO [Time-limited test {}] log.Log(170): Logging initialized @2850ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T05:14:43,229 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:14:43,309 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:14:43,336 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:14:43,337 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:14:43,339 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T05:14:43,359 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:14:43,366 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:14:43,368 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:14:43,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/java.io.tmpdir/jetty-localhost-43311-hadoop-hdfs-3_4_1-tests_jar-_-any-6640395362274794216/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T05:14:43,603 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:43311} 2024-12-03T05:14:43,604 INFO [Time-limited test {}] server.Server(415): Started @3315ms 2024-12-03T05:14:44,145 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:14:44,152 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:14:44,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:14:44,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:14:44,153 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T05:14:44,154 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:14:44,154 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:14:44,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/java.io.tmpdir/jetty-localhost-44887-hadoop-hdfs-3_4_1-tests_jar-_-any-7530958919685023554/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:14:44,265 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:44887} 2024-12-03T05:14:44,265 INFO [Time-limited test {}] server.Server(415): Started @3977ms 2024-12-03T05:14:44,315 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T05:14:45,317 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data2/current/BP-738945746-172.17.0.3-1733202882469/current, will proceed with Du for space computation calculation, 2024-12-03T05:14:45,317 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data1/current/BP-738945746-172.17.0.3-1733202882469/current, will proceed with Du for space computation calculation, 2024-12-03T05:14:45,351 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T05:14:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ceca08a65c06a4e with lease ID 0xb0fe2658a0ff56ab: Processing first storage report for DS-33e9aebc-abe2-4771-851c-8a711f911d0d from datanode DatanodeRegistration(127.0.0.1:42545, datanodeUuid=7ded0889-f37b-42aa-ac59-1df2d1988e00, infoPort=44635, infoSecurePort=0, ipcPort=35247, storageInfo=lv=-57;cid=testClusterID;nsid=1681540549;c=1733202882469) 2024-12-03T05:14:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ceca08a65c06a4e with lease ID 0xb0fe2658a0ff56ab: from storage DS-33e9aebc-abe2-4771-851c-8a711f911d0d node DatanodeRegistration(127.0.0.1:42545, datanodeUuid=7ded0889-f37b-42aa-ac59-1df2d1988e00, infoPort=44635, infoSecurePort=0, ipcPort=35247, storageInfo=lv=-57;cid=testClusterID;nsid=1681540549;c=1733202882469), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T05:14:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ceca08a65c06a4e with lease ID 0xb0fe2658a0ff56ab: Processing first storage report for DS-937856ba-68dd-40e8-b8f5-369350b1df74 from datanode DatanodeRegistration(127.0.0.1:42545, datanodeUuid=7ded0889-f37b-42aa-ac59-1df2d1988e00, infoPort=44635, infoSecurePort=0, ipcPort=35247, storageInfo=lv=-57;cid=testClusterID;nsid=1681540549;c=1733202882469) 2024-12-03T05:14:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ceca08a65c06a4e with lease ID 0xb0fe2658a0ff56ab: from storage DS-937856ba-68dd-40e8-b8f5-369350b1df74 node DatanodeRegistration(127.0.0.1:42545, datanodeUuid=7ded0889-f37b-42aa-ac59-1df2d1988e00, infoPort=44635, infoSecurePort=0, ipcPort=35247, storageInfo=lv=-57;cid=testClusterID;nsid=1681540549;c=1733202882469), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T05:14:45,484 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d 2024-12-03T05:14:45,631 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/zookeeper_0, clientPort=50539, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T05:14:45,643 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50539 2024-12-03T05:14:45,658 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:45,663 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:45,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741825_1001 (size=7) 2024-12-03T05:14:46,271 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd with version=8 2024-12-03T05:14:46,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/hbase-staging 2024-12-03T05:14:46,348 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T05:14:46,537 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:14:46,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:46,546 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:46,550 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:14:46,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:46,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:14:46,668 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T05:14:46,717 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T05:14:46,724 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T05:14:46,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:14:46,747 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4147 (auto-detected) 2024-12-03T05:14:46,748 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T05:14:46,767 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44191 2024-12-03T05:14:46,793 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44191 connecting to ZooKeeper ensemble=127.0.0.1:50539 2024-12-03T05:14:46,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:441910x0, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:14:46,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44191-0x101999d37940000 connected 2024-12-03T05:14:46,986 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:46,989 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:47,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:14:47,005 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd, hbase.cluster.distributed=false 2024-12-03T05:14:47,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:14:47,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44191 2024-12-03T05:14:47,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44191 2024-12-03T05:14:47,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44191 2024-12-03T05:14:47,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44191 2024-12-03T05:14:47,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44191 2024-12-03T05:14:47,128 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:14:47,129 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:47,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:47,130 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:14:47,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:14:47,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:14:47,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T05:14:47,135 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:14:47,136 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37863 2024-12-03T05:14:47,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37863 connecting to ZooKeeper ensemble=127.0.0.1:50539 2024-12-03T05:14:47,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:47,142 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:47,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378630x0, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:14:47,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378630x0, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:14:47,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37863-0x101999d37940001 connected 2024-12-03T05:14:47,157 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T05:14:47,163 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T05:14:47,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T05:14:47,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:14:47,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37863 2024-12-03T05:14:47,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37863 2024-12-03T05:14:47,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37863 2024-12-03T05:14:47,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37863 2024-12-03T05:14:47,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37863 2024-12-03T05:14:47,186 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e3c6570475b6:44191 2024-12-03T05:14:47,187 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e3c6570475b6,44191,1733202886389 2024-12-03T05:14:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:14:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:14:47,197 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e3c6570475b6,44191,1733202886389 2024-12-03T05:14:47,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T05:14:47,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:47,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:47,227 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T05:14:47,228 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e3c6570475b6,44191,1733202886389 from backup master directory 2024-12-03T05:14:47,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:14:47,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e3c6570475b6,44191,1733202886389 2024-12-03T05:14:47,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:14:47,238 WARN [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:14:47,238 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e3c6570475b6,44191,1733202886389 2024-12-03T05:14:47,240 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T05:14:47,242 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T05:14:47,292 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/hbase.id] with ID: a19201d6-9b21-461c-a6a9-e8f3e280f2f2 2024-12-03T05:14:47,292 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/.tmp/hbase.id 2024-12-03T05:14:47,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741826_1002 (size=42) 2024-12-03T05:14:47,703 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/.tmp/hbase.id]:[hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/hbase.id] 2024-12-03T05:14:47,757 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:14:47,763 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T05:14:47,781 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T05:14:47,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:47,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:47,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741827_1003 (size=196) 2024-12-03T05:14:48,267 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:14:48,269 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T05:14:48,284 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:48,288 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:14:48,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741828_1004 (size=1189) 2024-12-03T05:14:48,730 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store 2024-12-03T05:14:48,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741829_1005 (size=34) 2024-12-03T05:14:49,158 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T05:14:49,165 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:49,166 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T05:14:49,166 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:14:49,167 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:14:49,168 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T05:14:49,168 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:14:49,169 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:14:49,170 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733202889166Disabling compacts and flushes for region at 1733202889166Disabling writes for close at 1733202889168 (+2 ms)Writing region close event to WAL at 1733202889168Closed at 1733202889169 (+1 ms) 2024-12-03T05:14:49,172 WARN [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/.initializing 2024-12-03T05:14:49,173 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/WALs/e3c6570475b6,44191,1733202886389 2024-12-03T05:14:49,181 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:14:49,194 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C44191%2C1733202886389, suffix=, logDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/WALs/e3c6570475b6,44191,1733202886389, archiveDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/oldWALs, maxLogs=10 2024-12-03T05:14:49,215 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/WALs/e3c6570475b6,44191,1733202886389/e3c6570475b6%2C44191%2C1733202886389.1733202889198, exclude list is [], retry=0 2024-12-03T05:14:49,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42545,DS-33e9aebc-abe2-4771-851c-8a711f911d0d,DISK] 2024-12-03T05:14:49,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T05:14:49,271 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/WALs/e3c6570475b6,44191,1733202886389/e3c6570475b6%2C44191%2C1733202886389.1733202889198 2024-12-03T05:14:49,272 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44635:44635)] 2024-12-03T05:14:49,273 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:14:49,273 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:49,276 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,277 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T05:14:49,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:49,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T05:14:49,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:49,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T05:14:49,348 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:49,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,352 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T05:14:49,352 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:49,353 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,357 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,358 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,362 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,363 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,367 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T05:14:49,372 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:14:49,376 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:14:49,377 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73026008, jitterRate=0.0881723165512085}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T05:14:49,383 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733202889288Initializing all the Stores at 1733202889290 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202889290Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202889291 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202889291Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202889291Cleaning up temporary data from old regions at 1733202889363 (+72 ms)Region opened successfully at 1733202889383 (+20 ms) 2024-12-03T05:14:49,384 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T05:14:49,414 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1919649a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:14:49,444 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T05:14:49,453 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T05:14:49,453 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T05:14:49,455 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T05:14:49,456 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T05:14:49,460 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T05:14:49,461 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T05:14:49,487 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T05:14:49,496 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T05:14:49,564 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T05:14:49,569 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T05:14:49,572 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T05:14:49,584 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T05:14:49,587 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T05:14:49,591 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T05:14:49,603 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T05:14:49,605 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T05:14:49,615 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T05:14:49,637 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T05:14:49,647 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T05:14:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:14:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:14:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,662 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e3c6570475b6,44191,1733202886389, sessionid=0x101999d37940000, setting cluster-up flag (Was=false) 2024-12-03T05:14:49,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,721 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T05:14:49,725 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3c6570475b6,44191,1733202886389 2024-12-03T05:14:49,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:49,784 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T05:14:49,787 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3c6570475b6,44191,1733202886389 2024-12-03T05:14:49,794 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T05:14:49,870 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T05:14:49,878 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(746): ClusterId : a19201d6-9b21-461c-a6a9-e8f3e280f2f2 2024-12-03T05:14:49,880 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T05:14:49,880 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T05:14:49,886 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T05:14:49,892 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T05:14:49,892 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T05:14:49,893 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e3c6570475b6,44191,1733202886389 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T05:14:49,901 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T05:14:49,901 DEBUG [RS:0;e3c6570475b6:37863 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9214a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:14:49,902 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:14:49,902 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:14:49,902 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:14:49,902 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:14:49,903 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e3c6570475b6:0, corePoolSize=10, maxPoolSize=10 2024-12-03T05:14:49,903 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:49,903 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:14:49,903 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:49,908 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733202919908 2024-12-03T05:14:49,910 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T05:14:49,910 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:14:49,910 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T05:14:49,911 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T05:14:49,915 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T05:14:49,915 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,915 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T05:14:49,916 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T05:14:49,916 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T05:14:49,916 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e3c6570475b6:37863 2024-12-03T05:14:49,916 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T05:14:49,917 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:49,919 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T05:14:49,919 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T05:14:49,920 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T05:14:49,923 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T05:14:49,923 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(2659): reportForDuty to master=e3c6570475b6,44191,1733202886389 with port=37863, startcode=1733202887099 2024-12-03T05:14:49,924 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T05:14:49,924 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T05:14:49,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741831_1007 (size=1321) 2024-12-03T05:14:49,941 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T05:14:49,941 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:14:49,941 DEBUG [RS:0;e3c6570475b6:37863 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T05:14:49,947 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T05:14:49,948 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T05:14:49,953 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733202889949,5,FailOnTimeoutGroup] 2024-12-03T05:14:49,954 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733202889953,5,FailOnTimeoutGroup] 2024-12-03T05:14:49,954 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:49,954 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T05:14:49,955 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:49,955 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:49,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741832_1008 (size=32) 2024-12-03T05:14:49,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:49,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T05:14:49,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T05:14:49,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:49,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T05:14:49,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T05:14:49,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:49,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T05:14:49,984 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38517, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T05:14:49,987 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T05:14:49,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:49,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T05:14:49,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44191 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e3c6570475b6,37863,1733202887099 2024-12-03T05:14:49,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T05:14:49,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:49,994 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44191 {}] master.ServerManager(517): Registering regionserver=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:49,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:49,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T05:14:50,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740 2024-12-03T05:14:50,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740 2024-12-03T05:14:50,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T05:14:50,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T05:14:50,012 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:14:50,015 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:14:50,016 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46669 2024-12-03T05:14:50,016 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T05:14:50,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T05:14:50,022 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:14:50,023 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74410583, jitterRate=0.10880409181118011}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:14:50,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733202889960Initializing all the Stores at 1733202889964 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202889964Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202889965 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202889965Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202889965Cleaning up temporary data from old regions at 1733202890010 (+45 ms)Region opened successfully at 1733202890025 (+15 ms) 2024-12-03T05:14:50,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T05:14:50,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:14:50,026 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T05:14:50,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T05:14:50,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T05:14:50,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T05:14:50,027 DEBUG [RS:0;e3c6570475b6:37863 {}] zookeeper.ZKUtil(111): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,027 WARN [RS:0;e3c6570475b6:37863 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:14:50,027 INFO [RS:0;e3c6570475b6:37863 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:14:50,027 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,028 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T05:14:50,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733202890026Disabling compacts and flushes for region at 1733202890026Disabling writes for close at 1733202890026Writing region close event to WAL at 1733202890027 (+1 ms)Closed at 1733202890028 (+1 ms) 2024-12-03T05:14:50,030 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e3c6570475b6,37863,1733202887099] 2024-12-03T05:14:50,031 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:14:50,031 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T05:14:50,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T05:14:50,048 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T05:14:50,051 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T05:14:50,054 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T05:14:50,064 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T05:14:50,069 INFO [RS:0;e3c6570475b6:37863 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:14:50,069 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,070 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T05:14:50,076 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T05:14:50,078 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,078 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,078 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,078 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,079 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,079 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,079 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:14:50,079 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,079 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:14:50,080 DEBUG [RS:0;e3c6570475b6:37863 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:14:50,081 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,082 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,082 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,082 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,082 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,082 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37863,1733202887099-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:14:50,102 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T05:14:50,104 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37863,1733202887099-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,105 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,105 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.Replication(171): e3c6570475b6,37863,1733202887099 started 2024-12-03T05:14:50,121 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:50,122 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1482): Serving as e3c6570475b6,37863,1733202887099, RpcServer on e3c6570475b6/172.17.0.3:37863, sessionid=0x101999d37940001 2024-12-03T05:14:50,122 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T05:14:50,122 DEBUG [RS:0;e3c6570475b6:37863 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,122 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,37863,1733202887099' 2024-12-03T05:14:50,123 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T05:14:50,123 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T05:14:50,124 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T05:14:50,124 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T05:14:50,124 DEBUG [RS:0;e3c6570475b6:37863 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,124 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,37863,1733202887099' 2024-12-03T05:14:50,125 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T05:14:50,125 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T05:14:50,126 DEBUG [RS:0;e3c6570475b6:37863 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T05:14:50,126 INFO [RS:0;e3c6570475b6:37863 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T05:14:50,126 INFO [RS:0;e3c6570475b6:37863 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T05:14:50,203 WARN [e3c6570475b6:44191 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T05:14:50,241 INFO [RS:0;e3c6570475b6:37863 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:14:50,243 INFO [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C37863%2C1733202887099, suffix=, logDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099, archiveDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/oldWALs, maxLogs=32 2024-12-03T05:14:50,259 DEBUG [RS:0;e3c6570475b6:37863 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099/e3c6570475b6%2C37863%2C1733202887099.1733202890245, exclude list is [], retry=0 2024-12-03T05:14:50,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42545,DS-33e9aebc-abe2-4771-851c-8a711f911d0d,DISK] 2024-12-03T05:14:50,268 INFO [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099/e3c6570475b6%2C37863%2C1733202887099.1733202890245 2024-12-03T05:14:50,269 DEBUG [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44635:44635)] 2024-12-03T05:14:50,457 DEBUG [e3c6570475b6:44191 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T05:14:50,468 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,474 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3c6570475b6,37863,1733202887099, state=OPENING 2024-12-03T05:14:50,531 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T05:14:50,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:50,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:14:50,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:14:50,545 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:14:50,548 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T05:14:50,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:14:50,726 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:14:50,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:14:50,744 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T05:14:50,744 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:14:50,745 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T05:14:50,748 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C37863%2C1733202887099.meta, suffix=.meta, logDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099, archiveDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/oldWALs, maxLogs=32 2024-12-03T05:14:50,761 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099/e3c6570475b6%2C37863%2C1733202887099.meta.1733202890750.meta, exclude list is [], retry=0 2024-12-03T05:14:50,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42545,DS-33e9aebc-abe2-4771-851c-8a711f911d0d,DISK] 2024-12-03T05:14:50,768 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/WALs/e3c6570475b6,37863,1733202887099/e3c6570475b6%2C37863%2C1733202887099.meta.1733202890750.meta 2024-12-03T05:14:50,769 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44635:44635)] 2024-12-03T05:14:50,769 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:14:50,771 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T05:14:50,773 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T05:14:50,777 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T05:14:50,780 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T05:14:50,781 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:50,781 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T05:14:50,781 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T05:14:50,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T05:14:50,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T05:14:50,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:50,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:50,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T05:14:50,788 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T05:14:50,788 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:50,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:50,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T05:14:50,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T05:14:50,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:50,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:50,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T05:14:50,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T05:14:50,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:50,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:14:50,794 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T05:14:50,795 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740 2024-12-03T05:14:50,798 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740 2024-12-03T05:14:50,800 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T05:14:50,800 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T05:14:50,801 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:14:50,804 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T05:14:50,806 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64994569, jitterRate=-0.03150545060634613}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:14:50,806 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T05:14:50,808 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733202890782Writing region info on filesystem at 1733202890782Initializing all the Stores at 1733202890783 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202890784 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202890784Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202890784Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733202890784Cleaning up temporary data from old regions at 1733202890800 (+16 ms)Running coprocessor post-open hooks at 1733202890807 (+7 ms)Region opened successfully at 1733202890807 2024-12-03T05:14:50,814 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733202890717 2024-12-03T05:14:50,824 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T05:14:50,824 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T05:14:50,827 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,829 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3c6570475b6,37863,1733202887099, state=OPEN 2024-12-03T05:14:50,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:14:50,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:14:50,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:14:50,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:14:50,964 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:50,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T05:14:50,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e3c6570475b6,37863,1733202887099 in 413 msec 2024-12-03T05:14:50,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T05:14:50,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 936 msec 2024-12-03T05:14:50,982 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:14:50,982 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T05:14:51,001 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:51,002 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:51,020 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:51,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49889, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:51,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2190 sec 2024-12-03T05:14:51,041 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733202891041, completionTime=-1 2024-12-03T05:14:51,044 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T05:14:51,044 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T05:14:51,065 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T05:14:51,065 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733202951065 2024-12-03T05:14:51,065 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733203011065 2024-12-03T05:14:51,065 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-03T05:14:51,067 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,068 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,068 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,069 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e3c6570475b6:44191, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,069 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,070 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,076 DEBUG [master/e3c6570475b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T05:14:51,100 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.862sec 2024-12-03T05:14:51,102 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T05:14:51,103 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T05:14:51,104 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T05:14:51,104 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T05:14:51,104 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T05:14:51,105 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:14:51,106 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T05:14:51,113 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T05:14:51,114 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T05:14:51,114 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,44191,1733202886389-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:14:51,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:51,193 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T05:14:51,193 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T05:14:51,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:51,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:51,204 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:51,214 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:51,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:51,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:51,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:51,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e3c6570475b6,44191,1733202886389 2024-12-03T05:14:51,270 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=3883 2024-12-03T05:14:51,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:14:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:14:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-12-03T05:14:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:14:51,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:14:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:14:51,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:14:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:14:51,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:14:51,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T05:14:51,316 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:51,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:14:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:51,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741835_1011 (size=1061) 2024-12-03T05:14:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:51,745 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:14:51,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741836_1012 (size=53) 2024-12-03T05:14:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:52,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:52,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc274659350fd9dda12032ec9405be21, disabling compactions & flushes 2024-12-03T05:14:52,159 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,160 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. after waiting 0 ms 2024-12-03T05:14:52,160 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,160 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,160 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc274659350fd9dda12032ec9405be21: Waiting for close lock at 1733202892159Disabling compacts and flushes for region at 1733202892159Disabling writes for close at 1733202892160 (+1 ms)Writing region close event to WAL at 1733202892160Closed at 1733202892160 2024-12-03T05:14:52,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:14:52,172 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202892164"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202892164"}]},"ts":"1733202892164"} 2024-12-03T05:14:52,176 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:14:52,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:14:52,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202892177"}]},"ts":"1733202892177"} 2024-12-03T05:14:52,184 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:14:52,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, ASSIGN}] 2024-12-03T05:14:52,189 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, ASSIGN 2024-12-03T05:14:52,191 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:14:52,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fc274659350fd9dda12032ec9405be21, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:52,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, ASSIGN because future has completed 2024-12-03T05:14:52,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:52,513 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,514 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:14:52,514 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,515 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:14:52,515 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,515 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,518 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,523 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:14:52,524 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc274659350fd9dda12032ec9405be21 columnFamilyName A 2024-12-03T05:14:52,524 DEBUG [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:52,525 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(327): Store=fc274659350fd9dda12032ec9405be21/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:52,525 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,528 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:14:52,528 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc274659350fd9dda12032ec9405be21 columnFamilyName B 2024-12-03T05:14:52,528 DEBUG [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:52,529 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(327): Store=fc274659350fd9dda12032ec9405be21/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:52,529 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,531 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:14:52,532 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc274659350fd9dda12032ec9405be21 columnFamilyName C 2024-12-03T05:14:52,532 DEBUG [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:14:52,533 INFO [StoreOpener-fc274659350fd9dda12032ec9405be21-1 {}] regionserver.HStore(327): Store=fc274659350fd9dda12032ec9405be21/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:14:52,533 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,533 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,535 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,535 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,537 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,537 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,538 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:14:52,540 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,544 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:14:52,545 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fc274659350fd9dda12032ec9405be21; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69959297, jitterRate=0.042474761605262756}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:14:52,545 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:52,546 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fc274659350fd9dda12032ec9405be21: Running coprocessor pre-open hook at 1733202892515Writing region info on filesystem at 1733202892515Initializing all the Stores at 1733202892517 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202892517Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202892518 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202892518Cleaning up temporary data from old regions at 1733202892537 (+19 ms)Running coprocessor post-open hooks at 1733202892545 (+8 ms)Region opened successfully at 1733202892546 (+1 ms) 2024-12-03T05:14:52,548 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., pid=6, masterSystemTime=1733202892505 2024-12-03T05:14:52,551 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,551 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:52,552 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fc274659350fd9dda12032ec9405be21, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:52,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:14:52,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T05:14:52,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 in 208 msec 2024-12-03T05:14:52,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T05:14:52,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, ASSIGN in 376 msec 2024-12-03T05:14:52,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:14:52,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202892569"}]},"ts":"1733202892569"} 2024-12-03T05:14:52,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:14:52,574 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:14:52,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-12-03T05:14:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:14:53,472 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:14:53,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6900c275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,482 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a12729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,484 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:14:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:14:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@737a82e7 2024-12-03T05:14:53,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:14:53,496 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:14:53,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,502 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,509 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,524 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f535689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,527 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bffd9e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,535 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,539 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dae11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf5031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e54299b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ae3f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,555 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:14:53,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:14:53,558 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:14:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:14:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:14:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:14:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:14:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:14:53,560 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:14:53,564 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,565 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:14:53,566 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,568 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,568 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,569 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,573 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,573 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T05:14:53,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,579 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:14:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:14:53,581 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,606 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,621 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:14:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:14:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:14:53,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:14:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:14:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:53,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:14:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:53,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:53,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T05:14:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:53,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:53,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:53,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/94a8d12b25564549b5e1c493e8daaf45 is 50, key is test_row_0/A:col10/1733202893632/Put/seqid=0 2024-12-03T05:14:53,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741837_1013 (size=12001) 2024-12-03T05:14:53,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/94a8d12b25564549b5e1c493e8daaf45 2024-12-03T05:14:53,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:53,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:53,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202893821, max timestamp=1733202893853Num uniques 859; 2024-12-03T05:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202953854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:14:53,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202953849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202953850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:53,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202893842, max timestamp=1733202893853Num uniques 859; 2024-12-03T05:14:53,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T05:14:53,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202893842, max timestamp=1733202893858Num uniques 859; 2024-12-03T05:14:53,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:14:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:53,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/245dbd437f2d4adfbebd7548e0f5a808 is 50, key is test_row_0/B:col10/1733202893632/Put/seqid=0 2024-12-03T05:14:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:53,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T05:14:53,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:53,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:53,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:53,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:53,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741838_1014 (size=12001) 2024-12-03T05:14:53,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/245dbd437f2d4adfbebd7548e0f5a808 2024-12-03T05:14:53,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202953990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b3c76fb861754c8192a0d897683cc6ad is 50, key is test_row_0/C:col10/1733202893632/Put/seqid=0 2024-12-03T05:14:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202954006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741839_1015 (size=12001) 2024-12-03T05:14:54,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b3c76fb861754c8192a0d897683cc6ad 2024-12-03T05:14:54,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202954022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202954021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202954031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/94a8d12b25564549b5e1c493e8daaf45 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45 2024-12-03T05:14:54,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:14:54,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/245dbd437f2d4adfbebd7548e0f5a808 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808 2024-12-03T05:14:54,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:14:54,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b3c76fb861754c8192a0d897683cc6ad as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad 2024-12-03T05:14:54,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:14:54,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T05:14:54,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for fc274659350fd9dda12032ec9405be21 in 438ms, sequenceid=12, compaction requested=false 2024-12-03T05:14:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T05:14:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:54,123 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=129.31 KB heapSize=369.23 KB 2024-12-03T05:14:54,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:54,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b2861b8f887d47cfbdd34f61d2a42a72 is 50, key is test_row_0/A:col10/1733202893983/Put/seqid=0 2024-12-03T05:14:54,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741840_1016 (size=12001) 2024-12-03T05:14:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202954216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202954221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202954233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202954242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202954243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202954351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202954361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202954561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202954572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202954572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202954573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202954581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,585 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b2861b8f887d47cfbdd34f61d2a42a72 2024-12-03T05:14:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/faf460640f254851ab021bb1037d0185 is 50, key is test_row_0/B:col10/1733202893983/Put/seqid=0 2024-12-03T05:14:54,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741841_1017 (size=12001) 2024-12-03T05:14:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202954893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:54,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202954900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,039 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/faf460640f254851ab021bb1037d0185 2024-12-03T05:14:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/3a17f830918c4d99be243ec7cb5ed8d3 is 50, key is test_row_0/C:col10/1733202893983/Put/seqid=0 2024-12-03T05:14:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202955081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741842_1018 (size=12001) 2024-12-03T05:14:55,099 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/3a17f830918c4d99be243ec7cb5ed8d3 2024-12-03T05:14:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202955094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202955094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b2861b8f887d47cfbdd34f61d2a42a72 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72 2024-12-03T05:14:55,125 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:14:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/faf460640f254851ab021bb1037d0185 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185 2024-12-03T05:14:55,142 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:14:55,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/3a17f830918c4d99be243ec7cb5ed8d3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3 2024-12-03T05:14:55,165 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:14:55,168 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~369.18 KB/378040, currentSize=53.67 KB/54960 for fc274659350fd9dda12032ec9405be21 in 1044ms, sequenceid=48, compaction requested=false 2024-12-03T05:14:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T05:14:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T05:14:55,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T05:14:55,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5930 sec 2024-12-03T05:14:55,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6160 sec 2024-12-03T05:14:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:55,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:55,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/32628457e8084500bc1750533bc02b20 is 50, key is test_row_0/A:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:55,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741843_1019 (size=14341) 2024-12-03T05:14:55,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/32628457e8084500bc1750533bc02b20 2024-12-03T05:14:55,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/f38e51b10e28414e8f2aa557770e6153 is 50, key is test_row_0/B:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:55,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741844_1020 (size=12001) 2024-12-03T05:14:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202955529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202955534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/f38e51b10e28414e8f2aa557770e6153 2024-12-03T05:14:55,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/2b4d53df0e1a4196bdeff0163edb4855 is 50, key is test_row_0/C:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:55,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741845_1021 (size=12001) 2024-12-03T05:14:55,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/2b4d53df0e1a4196bdeff0163edb4855 2024-12-03T05:14:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202955641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/32628457e8084500bc1750533bc02b20 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20 2024-12-03T05:14:55,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202955650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20, entries=200, sequenceid=60, filesize=14.0 K 2024-12-03T05:14:55,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/f38e51b10e28414e8f2aa557770e6153 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153 2024-12-03T05:14:55,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T05:14:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/2b4d53df0e1a4196bdeff0163edb4855 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855 2024-12-03T05:14:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T05:14:55,721 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:14:55,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:14:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T05:14:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:55,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:14:55,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:14:55,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:14:55,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T05:14:55,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fc274659350fd9dda12032ec9405be21 in 312ms, sequenceid=60, compaction requested=true 2024-12-03T05:14:55,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:55,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:14:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:14:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:14:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:55,764 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:55,765 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:55,768 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:55,770 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:14:55,771 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:55,772 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.2 K 2024-12-03T05:14:55,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 245dbd437f2d4adfbebd7548e0f5a808, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202893632 2024-12-03T05:14:55,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting faf460640f254851ab021bb1037d0185, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202893969 2024-12-03T05:14:55,776 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f38e51b10e28414e8f2aa557770e6153, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:55,784 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:55,784 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:14:55,784 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:55,784 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=37.4 K 2024-12-03T05:14:55,788 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a8d12b25564549b5e1c493e8daaf45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202893632 2024-12-03T05:14:55,791 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2861b8f887d47cfbdd34f61d2a42a72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202893969 2024-12-03T05:14:55,793 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32628457e8084500bc1750533bc02b20, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:55,844 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:55,845 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c2b395529f8145cdb5ac32f5dd5396eb is 50, key is test_row_0/B:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:55,870 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:55,871 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/68e84851e4af43639475e64795472941 is 50, key is test_row_0/A:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:55,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202895510, max timestamp=1733202895857Num uniques 859; 2024-12-03T05:14:55,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:55,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:55,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:14:55,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741846_1022 (size=12104) 2024-12-03T05:14:55,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:55,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:55,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/4a6c2436b6c04504ad9e7fe19c1784f6 is 50, key is test_row_0/A:col10/1733202895864/Put/seqid=0 2024-12-03T05:14:55,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202955947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202955950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:55,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741847_1023 (size=12104) 2024-12-03T05:14:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741848_1024 (size=12001) 2024-12-03T05:14:56,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202956062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202956062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202956101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,111 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:14:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202956110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,115 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202956121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,128 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:14:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202956272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202956283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,298 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:14:56,317 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c2b395529f8145cdb5ac32f5dd5396eb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2b395529f8145cdb5ac32f5dd5396eb 2024-12-03T05:14:56,354 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into c2b395529f8145cdb5ac32f5dd5396eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:56,354 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:56,357 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202895763; duration=0sec 2024-12-03T05:14:56,358 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:56,358 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:14:56,358 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:56,367 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:56,367 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:14:56,367 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,368 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.2 K 2024-12-03T05:14:56,370 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c76fb861754c8192a0d897683cc6ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202893632 2024-12-03T05:14:56,371 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a17f830918c4d99be243ec7cb5ed8d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202893969 2024-12-03T05:14:56,373 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4d53df0e1a4196bdeff0163edb4855, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:56,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/4a6c2436b6c04504ad9e7fe19c1784f6 2024-12-03T05:14:56,393 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/68e84851e4af43639475e64795472941 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/68e84851e4af43639475e64795472941 2024-12-03T05:14:56,418 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into 68e84851e4af43639475e64795472941(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:56,419 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:56,419 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202895744; duration=0sec 2024-12-03T05:14:56,419 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:56,419 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:14:56,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/0c986c6745504a70b5a7dfbcc625af3a is 50, key is test_row_0/B:col10/1733202895864/Put/seqid=0 2024-12-03T05:14:56,441 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:56,442 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/91a0683ac5bc4a7d9b748b1b9ef0e56f is 50, key is test_row_0/C:col10/1733202895424/Put/seqid=0 2024-12-03T05:14:56,470 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T05:14:56,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741849_1025 (size=12104) 2024-12-03T05:14:56,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741850_1026 (size=12001) 2024-12-03T05:14:56,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/0c986c6745504a70b5a7dfbcc625af3a 2024-12-03T05:14:56,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/a2de992963f44c66b80fc98ee91e5ed2 is 50, key is test_row_0/C:col10/1733202895864/Put/seqid=0 2024-12-03T05:14:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202956583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202956601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:56,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741851_1027 (size=12001) 2024-12-03T05:14:56,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T05:14:56,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T05:14:56,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T05:14:56,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T05:14:56,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T05:14:56,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T05:14:56,720 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T05:14:56,720 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T05:14:56,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:56,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:56,911 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/91a0683ac5bc4a7d9b748b1b9ef0e56f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/91a0683ac5bc4a7d9b748b1b9ef0e56f 2024-12-03T05:14:56,937 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into 91a0683ac5bc4a7d9b748b1b9ef0e56f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:56,937 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:56,937 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202895763; duration=0sec 2024-12-03T05:14:56,938 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:56,938 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:14:57,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/a2de992963f44c66b80fc98ee91e5ed2 2024-12-03T05:14:57,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/4a6c2436b6c04504ad9e7fe19c1784f6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6 2024-12-03T05:14:57,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T05:14:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/0c986c6745504a70b5a7dfbcc625af3a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a 2024-12-03T05:14:57,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T05:14:57,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/a2de992963f44c66b80fc98ee91e5ed2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2 2024-12-03T05:14:57,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T05:14:57,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for fc274659350fd9dda12032ec9405be21 in 1207ms, sequenceid=82, compaction requested=false 2024-12-03T05:14:57,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:57,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/da6748a8c1924a118c932d589487ae1a is 50, key is test_row_0/A:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741852_1028 (size=12001) 2024-12-03T05:14:57,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/da6748a8c1924a118c932d589487ae1a 2024-12-03T05:14:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202957211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202957211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/6e785e7171d04803803d497ac2c7f21f is 50, key is test_row_0/B:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741853_1029 (size=12001) 2024-12-03T05:14:57,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/6e785e7171d04803803d497ac2c7f21f 2024-12-03T05:14:57,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/5ccee404b02e471189f629ecb19b1403 is 50, key is test_row_0/C:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741854_1030 (size=12001) 2024-12-03T05:14:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202957321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202957321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202957532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202957532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:14:57,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/5ccee404b02e471189f629ecb19b1403 2024-12-03T05:14:57,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/da6748a8c1924a118c932d589487ae1a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a 2024-12-03T05:14:57,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T05:14:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/6e785e7171d04803803d497ac2c7f21f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f 2024-12-03T05:14:57,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T05:14:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/5ccee404b02e471189f629ecb19b1403 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403 2024-12-03T05:14:57,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T05:14:57,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for fc274659350fd9dda12032ec9405be21 in 679ms, sequenceid=101, compaction requested=true 2024-12-03T05:14:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:14:57,787 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:14:57,787 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:57,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:14:57,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:57,790 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:57,790 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:57,790 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:14:57,790 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:14:57,790 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,790 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,790 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/68e84851e4af43639475e64795472941, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.3 K 2024-12-03T05:14:57,790 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2b395529f8145cdb5ac32f5dd5396eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.3 K 2024-12-03T05:14:57,791 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b395529f8145cdb5ac32f5dd5396eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:57,793 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e84851e4af43639475e64795472941, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:57,794 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c986c6745504a70b5a7dfbcc625af3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733202895524 2024-12-03T05:14:57,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T05:14:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,796 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T05:14:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:57,797 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6c2436b6c04504ad9e7fe19c1784f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733202895524 2024-12-03T05:14:57,798 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e785e7171d04803803d497ac2c7f21f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:57,799 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6748a8c1924a118c932d589487ae1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:57,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/9e499224e5c649bfae33626a80bf4c6f is 50, key is test_row_0/A:col10/1733202897206/Put/seqid=0 2024-12-03T05:14:57,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741855_1031 (size=12001) 2024-12-03T05:14:57,829 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/9e499224e5c649bfae33626a80bf4c6f 2024-12-03T05:14:57,844 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:57,845 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bada4a6d4290434ab2361452d4dd66e7 is 50, key is test_row_0/B:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,847 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#19 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:57,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:14:57,847 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a8c1d514b3124c0ebc33770cf95fed02 is 50, key is test_row_0/A:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2f413b57b82545b98aa121bc086ea552 is 50, key is test_row_0/B:col10/1733202897206/Put/seqid=0 2024-12-03T05:14:57,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741857_1033 (size=12207) 2024-12-03T05:14:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:57,885 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a8c1d514b3124c0ebc33770cf95fed02 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a8c1d514b3124c0ebc33770cf95fed02 2024-12-03T05:14:57,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741856_1032 (size=12207) 2024-12-03T05:14:57,899 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into a8c1d514b3124c0ebc33770cf95fed02(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:57,900 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:57,900 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202897787; duration=0sec 2024-12-03T05:14:57,900 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:57,900 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:14:57,900 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202957894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,903 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,904 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:14:57,904 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:57,904 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/91a0683ac5bc4a7d9b748b1b9ef0e56f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.3 K 2024-12-03T05:14:57,905 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a0683ac5bc4a7d9b748b1b9ef0e56f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733202894163 2024-12-03T05:14:57,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202957899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:57,906 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2de992963f44c66b80fc98ee91e5ed2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733202895524 2024-12-03T05:14:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:57,908 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ccee404b02e471189f629ecb19b1403, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741858_1034 (size=12001) 2024-12-03T05:14:57,913 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2f413b57b82545b98aa121bc086ea552 2024-12-03T05:14:57,930 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:57,930 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b307e01dd37a4069bd796dd88d5c5724 is 50, key is test_row_0/C:col10/1733202895936/Put/seqid=0 2024-12-03T05:14:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e02ca0cc0bd04428848fa56b0fe68a8d is 50, key is test_row_0/C:col10/1733202897206/Put/seqid=0 2024-12-03T05:14:57,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741859_1035 (size=12207) 2024-12-03T05:14:57,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741860_1036 (size=12001) 2024-12-03T05:14:57,991 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e02ca0cc0bd04428848fa56b0fe68a8d 2024-12-03T05:14:57,992 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b307e01dd37a4069bd796dd88d5c5724 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b307e01dd37a4069bd796dd88d5c5724 2024-12-03T05:14:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/9e499224e5c649bfae33626a80bf4c6f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f 2024-12-03T05:14:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202958010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202958012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,018 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into b307e01dd37a4069bd796dd88d5c5724(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:58,018 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,018 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202897787; duration=0sec 2024-12-03T05:14:58,019 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,019 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:14:58,019 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T05:14:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2f413b57b82545b98aa121bc086ea552 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552 2024-12-03T05:14:58,038 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T05:14:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e02ca0cc0bd04428848fa56b0fe68a8d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d 2024-12-03T05:14:58,056 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T05:14:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc274659350fd9dda12032ec9405be21 in 267ms, sequenceid=118, compaction requested=false 2024-12-03T05:14:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T05:14:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T05:14:58,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T05:14:58,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3320 sec 2024-12-03T05:14:58,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3530 sec 2024-12-03T05:14:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:58,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/009810ddc2d14378a8ca42c711a137c7 is 50, key is test_row_0/A:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741861_1037 (size=14541) 2024-12-03T05:14:58,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/009810ddc2d14378a8ca42c711a137c7 2024-12-03T05:14:58,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/98d2d9dc961f49688aab01807cdb1714 is 50, key is test_row_0/B:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202958185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202958185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202958194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741862_1038 (size=12151) 2024-12-03T05:14:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202958222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202958222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/98d2d9dc961f49688aab01807cdb1714 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/352f3bfd91894a448abed927fd5fbb2b is 50, key is test_row_0/C:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202958301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741863_1039 (size=12151) 2024-12-03T05:14:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202958301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/352f3bfd91894a448abed927fd5fbb2b 2024-12-03T05:14:58,313 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bada4a6d4290434ab2361452d4dd66e7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bada4a6d4290434ab2361452d4dd66e7 2024-12-03T05:14:58,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202958310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/009810ddc2d14378a8ca42c711a137c7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7 2024-12-03T05:14:58,328 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into bada4a6d4290434ab2361452d4dd66e7(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:58,329 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,329 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202897787; duration=0sec 2024-12-03T05:14:58,330 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,330 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:14:58,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7, entries=200, sequenceid=136, filesize=14.2 K 2024-12-03T05:14:58,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/98d2d9dc961f49688aab01807cdb1714 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714 2024-12-03T05:14:58,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T05:14:58,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/352f3bfd91894a448abed927fd5fbb2b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b 2024-12-03T05:14:58,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T05:14:58,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc274659350fd9dda12032ec9405be21 in 245ms, sequenceid=136, compaction requested=true 2024-12-03T05:14:58,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,381 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:14:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:14:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,381 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:58,383 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:58,383 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:14:58,383 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:58,383 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:14:58,383 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:58,383 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:58,383 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a8c1d514b3124c0ebc33770cf95fed02, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=37.8 K 2024-12-03T05:14:58,383 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bada4a6d4290434ab2361452d4dd66e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.5 K 2024-12-03T05:14:58,384 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bada4a6d4290434ab2361452d4dd66e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:58,384 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c1d514b3124c0ebc33770cf95fed02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:58,385 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e499224e5c649bfae33626a80bf4c6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733202897195 2024-12-03T05:14:58,385 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f413b57b82545b98aa121bc086ea552, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733202897195 2024-12-03T05:14:58,385 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d2d9dc961f49688aab01807cdb1714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:14:58,385 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 009810ddc2d14378a8ca42c711a137c7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:14:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:14:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:58,403 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:58,404 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b2565bbae7c6488cafcda43da601139f is 50, key is test_row_0/A:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,411 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:58,412 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/fbd0c8b48dbd4864b46d36eb52f6a9b6 is 50, key is test_row_0/B:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741865_1041 (size=12459) 2024-12-03T05:14:58,471 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b2565bbae7c6488cafcda43da601139f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2565bbae7c6488cafcda43da601139f 2024-12-03T05:14:58,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741864_1040 (size=12459) 2024-12-03T05:14:58,495 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into b2565bbae7c6488cafcda43da601139f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:58,495 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,495 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202898381; duration=0sec 2024-12-03T05:14:58,495 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:14:58,495 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:14:58,496 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:14:58,505 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:14:58,505 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:14:58,505 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:14:58,506 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b307e01dd37a4069bd796dd88d5c5724, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.5 K 2024-12-03T05:14:58,506 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b307e01dd37a4069bd796dd88d5c5724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733202895936 2024-12-03T05:14:58,507 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e02ca0cc0bd04428848fa56b0fe68a8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733202897195 2024-12-03T05:14:58,507 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352f3bfd91894a448abed927fd5fbb2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:14:58,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:58,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/34f4c6ef09e84b3c95197bf138aade31 is 50, key is test_row_0/A:col10/1733202898170/Put/seqid=0 2024-12-03T05:14:58,546 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:14:58,546 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b7da38bd058d45c3b4999e6f5b9e052e is 50, key is test_row_0/C:col10/1733202897885/Put/seqid=0 2024-12-03T05:14:58,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741866_1042 (size=12151) 2024-12-03T05:14:58,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741867_1043 (size=12459) 2024-12-03T05:14:58,570 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b7da38bd058d45c3b4999e6f5b9e052e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b7da38bd058d45c3b4999e6f5b9e052e 2024-12-03T05:14:58,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202958561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,583 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into b7da38bd058d45c3b4999e6f5b9e052e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:58,583 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,583 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202898381; duration=0sec 2024-12-03T05:14:58,583 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,584 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:14:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202958564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202958568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202958578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202958578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202958693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202958693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202958693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202958693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202958694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202958900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202958901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202958901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202958901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,912 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/fbd0c8b48dbd4864b46d36eb52f6a9b6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/fbd0c8b48dbd4864b46d36eb52f6a9b6 2024-12-03T05:14:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202958915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:58,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:58,929 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into fbd0c8b48dbd4864b46d36eb52f6a9b6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:14:58,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:58,930 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202898381; duration=0sec 2024-12-03T05:14:58,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:14:58,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:14:58,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/34f4c6ef09e84b3c95197bf138aade31 2024-12-03T05:14:58,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c7303a5089f54ffca32417a8338e4ada is 50, key is test_row_0/B:col10/1733202898170/Put/seqid=0 2024-12-03T05:14:59,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741868_1044 (size=12151) 2024-12-03T05:14:59,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202959211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202959211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202959212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202959221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202959233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c7303a5089f54ffca32417a8338e4ada 2024-12-03T05:14:59,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f3bcd1f71824489199bf5abdc366cfa8 is 50, key is test_row_0/C:col10/1733202898170/Put/seqid=0 2024-12-03T05:14:59,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741869_1045 (size=12151) 2024-12-03T05:14:59,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f3bcd1f71824489199bf5abdc366cfa8 2024-12-03T05:14:59,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/34f4c6ef09e84b3c95197bf138aade31 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31 2024-12-03T05:14:59,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31, entries=150, sequenceid=156, filesize=11.9 K 2024-12-03T05:14:59,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c7303a5089f54ffca32417a8338e4ada as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada 2024-12-03T05:14:59,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada, entries=150, sequenceid=156, filesize=11.9 K 2024-12-03T05:14:59,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f3bcd1f71824489199bf5abdc366cfa8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8 2024-12-03T05:14:59,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8, entries=150, sequenceid=156, filesize=11.9 K 2024-12-03T05:14:59,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for fc274659350fd9dda12032ec9405be21 in 1042ms, sequenceid=156, compaction requested=false 2024-12-03T05:14:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:14:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:14:59,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:14:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:14:59,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/2c7f2defd2f44869aab91ba9b4a10de2 is 50, key is test_row_0/A:col10/1733202899724/Put/seqid=0 2024-12-03T05:14:59,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202959766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202959768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202959773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202959771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202959774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741870_1046 (size=12151) 2024-12-03T05:14:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202959881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202959891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202959891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202959891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:14:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202959892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:14:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T05:14:59,901 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:14:59,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:14:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T05:14:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:14:59,904 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:14:59,905 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:14:59,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:15:00,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202960091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202960100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202960100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202960101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202960101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/2c7f2defd2f44869aab91ba9b4a10de2 2024-12-03T05:15:00,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3759ac7afaf4484690ae07ab02e8cbce is 50, key is test_row_0/B:col10/1733202899724/Put/seqid=0 2024-12-03T05:15:00,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:15:00,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741871_1047 (size=12151) 2024-12-03T05:15:00,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202960401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202960410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202960410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202960411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202960412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:15:00,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3759ac7afaf4484690ae07ab02e8cbce 2024-12-03T05:15:00,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/649dbcc4d48b40d0aeafc64d01fa8389 is 50, key is test_row_0/C:col10/1733202899724/Put/seqid=0 2024-12-03T05:15:00,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:00,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741872_1048 (size=12151) 2024-12-03T05:15:00,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/649dbcc4d48b40d0aeafc64d01fa8389 2024-12-03T05:15:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/2c7f2defd2f44869aab91ba9b4a10de2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2 2024-12-03T05:15:00,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2, entries=150, sequenceid=175, filesize=11.9 K 2024-12-03T05:15:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3759ac7afaf4484690ae07ab02e8cbce as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce 2024-12-03T05:15:00,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce, entries=150, sequenceid=175, filesize=11.9 K 2024-12-03T05:15:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/649dbcc4d48b40d0aeafc64d01fa8389 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389 2024-12-03T05:15:00,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389, entries=150, sequenceid=175, filesize=11.9 K 2024-12-03T05:15:00,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for fc274659350fd9dda12032ec9405be21 in 1045ms, sequenceid=175, compaction requested=true 2024-12-03T05:15:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:00,774 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:00,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:00,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:00,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:00,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:00,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:00,776 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:00,776 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,776 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/fbd0c8b48dbd4864b46d36eb52f6a9b6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.9 K 2024-12-03T05:15:00,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd0c8b48dbd4864b46d36eb52f6a9b6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:15:00,777 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:00,777 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:00,777 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,778 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2565bbae7c6488cafcda43da601139f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.9 K 2024-12-03T05:15:00,778 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c7303a5089f54ffca32417a8338e4ada, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733202898170 2024-12-03T05:15:00,779 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2565bbae7c6488cafcda43da601139f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:15:00,779 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3759ac7afaf4484690ae07ab02e8cbce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:00,780 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f4c6ef09e84b3c95197bf138aade31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733202898170 2024-12-03T05:15:00,781 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c7f2defd2f44869aab91ba9b4a10de2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:00,799 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:00,800 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:00,800 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7ca38243c9cb451697565566d42f19eb is 50, key is test_row_0/B:col10/1733202899724/Put/seqid=0 2024-12-03T05:15:00,800 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b497542d40b84663a75907174693f93c is 50, key is test_row_0/A:col10/1733202899724/Put/seqid=0 2024-12-03T05:15:00,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741873_1049 (size=12561) 2024-12-03T05:15:00,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T05:15:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T05:15:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:00,840 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7ca38243c9cb451697565566d42f19eb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7ca38243c9cb451697565566d42f19eb 2024-12-03T05:15:00,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741874_1050 (size=12561) 2024-12-03T05:15:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/bd5824c1ca3a41b09fc64096454e3d62 is 50, key is test_row_0/A:col10/1733202899770/Put/seqid=0 2024-12-03T05:15:00,855 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b497542d40b84663a75907174693f93c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b497542d40b84663a75907174693f93c 2024-12-03T05:15:00,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741875_1051 (size=12151) 2024-12-03T05:15:00,866 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into 7ca38243c9cb451697565566d42f19eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:00,866 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:00,867 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202900774; duration=0sec 2024-12-03T05:15:00,867 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:00,867 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:00,867 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:00,869 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/bd5824c1ca3a41b09fc64096454e3d62 2024-12-03T05:15:00,869 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:00,870 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:00,870 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:00,870 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b7da38bd058d45c3b4999e6f5b9e052e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=35.9 K 2024-12-03T05:15:00,870 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b7da38bd058d45c3b4999e6f5b9e052e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733202897885 2024-12-03T05:15:00,872 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bcd1f71824489199bf5abdc366cfa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733202898170 2024-12-03T05:15:00,873 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 649dbcc4d48b40d0aeafc64d01fa8389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:00,875 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into b497542d40b84663a75907174693f93c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:00,875 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:00,875 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202900774; duration=0sec 2024-12-03T05:15:00,876 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:00,876 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72f5f2c0d57c44558b152b9a176b7279 is 50, key is test_row_0/B:col10/1733202899770/Put/seqid=0 2024-12-03T05:15:00,898 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:00,899 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b12d34f28c3c474aa9421ee078d46cf0 is 50, key is test_row_0/C:col10/1733202899724/Put/seqid=0 2024-12-03T05:15:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:00,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741876_1052 (size=12151) 2024-12-03T05:15:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741877_1053 (size=12561) 2024-12-03T05:15:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202960967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:00,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202960985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202960992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202960992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202960998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:15:01,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202961102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202961111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202961112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202961112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202961121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202961310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202961320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202961331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202961331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202961333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,346 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72f5f2c0d57c44558b152b9a176b7279 2024-12-03T05:15:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7d26d39a27fd4477a4a2647e3ac673ef is 50, key is test_row_0/C:col10/1733202899770/Put/seqid=0 2024-12-03T05:15:01,370 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/b12d34f28c3c474aa9421ee078d46cf0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b12d34f28c3c474aa9421ee078d46cf0 2024-12-03T05:15:01,378 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into b12d34f28c3c474aa9421ee078d46cf0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:01,378 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:01,378 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202900775; duration=0sec 2024-12-03T05:15:01,379 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:01,379 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:01,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741878_1054 (size=12151) 2024-12-03T05:15:01,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202961620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202961631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202961641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202961641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202961641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:01,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7d26d39a27fd4477a4a2647e3ac673ef 2024-12-03T05:15:01,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/bd5824c1ca3a41b09fc64096454e3d62 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62 2024-12-03T05:15:01,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T05:15:01,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72f5f2c0d57c44558b152b9a176b7279 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279 2024-12-03T05:15:01,817 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T05:15:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7d26d39a27fd4477a4a2647e3ac673ef as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef 2024-12-03T05:15:01,833 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T05:15:01,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for fc274659350fd9dda12032ec9405be21 in 1002ms, sequenceid=191, compaction requested=false 2024-12-03T05:15:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T05:15:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T05:15:01,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T05:15:01,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-12-03T05:15:01,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9430 sec 2024-12-03T05:15:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T05:15:02,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:02,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T05:15:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:02,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:02,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:02,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:02,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:02,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:02,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6e94cb013fa74284a30a5b1c3eb886f5 is 50, key is test_row_0/A:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:02,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741879_1055 (size=14541) 2024-12-03T05:15:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:02,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202962161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202962161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202962163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202962165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202962165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202962270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202962281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202962280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202962281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202962481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202962491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202962491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202962491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6e94cb013fa74284a30a5b1c3eb886f5 2024-12-03T05:15:02,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72ae72889ff5421e9fe92a9e25471d86 is 50, key is test_row_0/B:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:02,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741880_1056 (size=12151) 2024-12-03T05:15:02,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72ae72889ff5421e9fe92a9e25471d86 2024-12-03T05:15:02,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/254fe6f6b2994cc5bdd0e7ce1de9f3df is 50, key is test_row_0/C:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:02,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741881_1057 (size=12151) 2024-12-03T05:15:02,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202962790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202962800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202962801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202962810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:02,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/254fe6f6b2994cc5bdd0e7ce1de9f3df 2024-12-03T05:15:02,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:02,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:03,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6e94cb013fa74284a30a5b1c3eb886f5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5 2024-12-03T05:15:03,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5, entries=200, sequenceid=213, filesize=14.2 K 2024-12-03T05:15:03,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/72ae72889ff5421e9fe92a9e25471d86 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86 2024-12-03T05:15:03,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T05:15:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/254fe6f6b2994cc5bdd0e7ce1de9f3df as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df 2024-12-03T05:15:03,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T05:15:03,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for fc274659350fd9dda12032ec9405be21 in 937ms, sequenceid=213, compaction requested=true 2024-12-03T05:15:03,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:03,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:03,072 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:03,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:03,073 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:03,074 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:03,074 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:03,074 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:03,075 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b497542d40b84663a75907174693f93c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=38.3 K 2024-12-03T05:15:03,074 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:03,082 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:03,082 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:03,082 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7ca38243c9cb451697565566d42f19eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.0 K 2024-12-03T05:15:03,083 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b497542d40b84663a75907174693f93c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:03,084 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca38243c9cb451697565566d42f19eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:03,084 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5824c1ca3a41b09fc64096454e3d62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733202899767 2024-12-03T05:15:03,085 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e94cb013fa74284a30a5b1c3eb886f5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:03,085 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f5f2c0d57c44558b152b9a176b7279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733202899767 2024-12-03T05:15:03,085 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ae72889ff5421e9fe92a9e25471d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:03,106 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:03,106 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/24508790506c4121af9f864029ca77aa is 50, key is test_row_0/B:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:03,111 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:03,112 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b3c5353a5e4d477888cde436d040928f is 50, key is test_row_0/A:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:03,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741883_1059 (size=12663) 2024-12-03T05:15:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741882_1058 (size=12663) 2024-12-03T05:15:03,132 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/b3c5353a5e4d477888cde436d040928f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b3c5353a5e4d477888cde436d040928f 2024-12-03T05:15:03,138 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into b3c5353a5e4d477888cde436d040928f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:03,138 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:03,138 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202903072; duration=0sec 2024-12-03T05:15:03,138 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:03,138 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:03,138 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:03,140 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:03,140 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:03,140 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:03,140 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b12d34f28c3c474aa9421ee078d46cf0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.0 K 2024-12-03T05:15:03,141 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12d34f28c3c474aa9421ee078d46cf0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733202898549 2024-12-03T05:15:03,141 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d26d39a27fd4477a4a2647e3ac673ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733202899767 2024-12-03T05:15:03,142 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254fe6f6b2994cc5bdd0e7ce1de9f3df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:03,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:03,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,162 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/c4d26604315141998d075ee25f167913 is 50, key is test_row_0/A:col10/1733202902156/Put/seqid=0 2024-12-03T05:15:03,163 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/a56efaab58e843889a68a54db7461574 is 50, key is test_row_0/C:col10/1733202900982/Put/seqid=0 2024-12-03T05:15:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741884_1060 (size=12151) 2024-12-03T05:15:03,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/c4d26604315141998d075ee25f167913 2024-12-03T05:15:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741885_1061 (size=12663) 2024-12-03T05:15:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:03,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7484251936ec435cbd9d39e65cb6fbde is 50, key is test_row_0/B:col10/1733202902156/Put/seqid=0 2024-12-03T05:15:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741886_1062 (size=12151) 2024-12-03T05:15:03,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202963282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202963300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202963311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202963311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202963320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202963390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,541 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/24508790506c4121af9f864029ca77aa as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/24508790506c4121af9f864029ca77aa 2024-12-03T05:15:03,550 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into 24508790506c4121af9f864029ca77aa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:03,550 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:03,550 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202903072; duration=0sec 2024-12-03T05:15:03,550 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:03,550 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202963600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:03,611 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/a56efaab58e843889a68a54db7461574 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a56efaab58e843889a68a54db7461574 2024-12-03T05:15:03,620 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7484251936ec435cbd9d39e65cb6fbde 2024-12-03T05:15:03,632 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into a56efaab58e843889a68a54db7461574(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:03,632 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:03,632 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202903073; duration=0sec 2024-12-03T05:15:03,632 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:03,632 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:03,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/55874f9cba4b409581d9bd1ef7b33bf3 is 50, key is test_row_0/C:col10/1733202902156/Put/seqid=0 2024-12-03T05:15:03,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741887_1063 (size=12151) 2024-12-03T05:15:03,657 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/55874f9cba4b409581d9bd1ef7b33bf3 2024-12-03T05:15:03,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/c4d26604315141998d075ee25f167913 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913 2024-12-03T05:15:03,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T05:15:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7484251936ec435cbd9d39e65cb6fbde as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde 2024-12-03T05:15:03,757 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T05:15:03,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/55874f9cba4b409581d9bd1ef7b33bf3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3 2024-12-03T05:15:03,771 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3, entries=150, sequenceid=228, filesize=11.9 K 2024-12-03T05:15:03,772 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fc274659350fd9dda12032ec9405be21 in 620ms, sequenceid=228, compaction requested=false 2024-12-03T05:15:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:03,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T05:15:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T05:15:03,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T05:15:03,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2024-12-03T05:15:03,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7490 sec 2024-12-03T05:15:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:03,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:03,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/0bc15588410e4616a8f3d54eed4008c8 is 50, key is test_row_0/A:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741888_1064 (size=12151) 2024-12-03T05:15:03,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/0bc15588410e4616a8f3d54eed4008c8 2024-12-03T05:15:03,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e28dfa8ef35845cb937c66f68318395b is 50, key is test_row_0/B:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741889_1065 (size=12151) 2024-12-03T05:15:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202963990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202964101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T05:15:04,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:04,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T05:15:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:04,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:04,225 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:04,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:04,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202964311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202964320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,322 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202964331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202964332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,333 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202964332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,338 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,340 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T05:15:04,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:04,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e28dfa8ef35845cb937c66f68318395b 2024-12-03T05:15:04,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/16739f1984814fbe986c1acce58cfb68 is 50, key is test_row_0/C:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741890_1066 (size=12151) 2024-12-03T05:15:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:04,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T05:15:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202964620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:04,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T05:15:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/16739f1984814fbe986c1acce58cfb68 2024-12-03T05:15:04,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/0bc15588410e4616a8f3d54eed4008c8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8 2024-12-03T05:15:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:04,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T05:15:04,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e28dfa8ef35845cb937c66f68318395b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b 2024-12-03T05:15:04,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T05:15:04,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/16739f1984814fbe986c1acce58cfb68 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68 2024-12-03T05:15:04,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T05:15:04,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:04,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T05:15:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:04,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fc274659350fd9dda12032ec9405be21 in 968ms, sequenceid=249, compaction requested=true 2024-12-03T05:15:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:04,883 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:04,883 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:04,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:04,885 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:04,885 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:04,885 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:04,885 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:04,885 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,885 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,885 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/24508790506c4121af9f864029ca77aa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.1 K 2024-12-03T05:15:04,885 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b3c5353a5e4d477888cde436d040928f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.1 K 2024-12-03T05:15:04,886 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c5353a5e4d477888cde436d040928f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:04,886 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 24508790506c4121af9f864029ca77aa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:04,887 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d26604315141998d075ee25f167913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733202902156 2024-12-03T05:15:04,887 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7484251936ec435cbd9d39e65cb6fbde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733202902156 2024-12-03T05:15:04,887 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc15588410e4616a8f3d54eed4008c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:04,887 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e28dfa8ef35845cb937c66f68318395b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:04,902 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:04,903 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:04,903 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/e3bcf2f053594eaea0cc57fca7dac1c9 is 50, key is test_row_0/A:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:04,903 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bc32aa5cc4634779a1ee2e6d451a0e11 is 50, key is test_row_0/B:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:04,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741891_1067 (size=12765) 2024-12-03T05:15:04,940 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/e3bcf2f053594eaea0cc57fca7dac1c9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/e3bcf2f053594eaea0cc57fca7dac1c9 2024-12-03T05:15:04,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741892_1068 (size=12765) 2024-12-03T05:15:04,954 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into e3bcf2f053594eaea0cc57fca7dac1c9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:04,955 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:04,955 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202904883; duration=0sec 2024-12-03T05:15:04,955 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:04,955 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:04,955 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:04,958 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bc32aa5cc4634779a1ee2e6d451a0e11 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bc32aa5cc4634779a1ee2e6d451a0e11 2024-12-03T05:15:04,959 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:04,959 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:04,959 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:04,959 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a56efaab58e843889a68a54db7461574, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.1 K 2024-12-03T05:15:04,962 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56efaab58e843889a68a54db7461574, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733202900962 2024-12-03T05:15:04,968 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55874f9cba4b409581d9bd1ef7b33bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733202902156 2024-12-03T05:15:04,971 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16739f1984814fbe986c1acce58cfb68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:04,981 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into bc32aa5cc4634779a1ee2e6d451a0e11(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:04,981 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:04,981 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202904883; duration=0sec 2024-12-03T05:15:04,981 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:04,981 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:05,026 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:05,026 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/6e9f649f996447bb897ce54e42c257be is 50, key is test_row_0/C:col10/1733202903253/Put/seqid=0 2024-12-03T05:15:05,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T05:15:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:05,040 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:05,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/54e6901040014eeb850019f257540519 is 50, key is test_row_0/A:col10/1733202903949/Put/seqid=0 2024-12-03T05:15:05,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741893_1069 (size=12765) 2024-12-03T05:15:05,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741894_1070 (size=12301) 2024-12-03T05:15:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:05,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:05,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202965256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202965370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:05,478 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/6e9f649f996447bb897ce54e42c257be as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6e9f649f996447bb897ce54e42c257be 2024-12-03T05:15:05,484 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into 6e9f649f996447bb897ce54e42c257be(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:05,485 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:05,485 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202904884; duration=0sec 2024-12-03T05:15:05,485 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:05,485 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:05,488 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/54e6901040014eeb850019f257540519 2024-12-03T05:15:05,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7d6b906bbcd241aca5dfdc8cab68c814 is 50, key is test_row_0/B:col10/1733202903949/Put/seqid=0 2024-12-03T05:15:05,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741895_1071 (size=12301) 2024-12-03T05:15:05,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7d6b906bbcd241aca5dfdc8cab68c814 2024-12-03T05:15:05,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/6acc55d0e7a0459ab96a10b8ec893a27 is 50, key is test_row_0/C:col10/1733202903949/Put/seqid=0 2024-12-03T05:15:05,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741896_1072 (size=12301) 2024-12-03T05:15:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202965580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202965890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:05,972 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/6acc55d0e7a0459ab96a10b8ec893a27 2024-12-03T05:15:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/54e6901040014eeb850019f257540519 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519 2024-12-03T05:15:05,990 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519, entries=150, sequenceid=264, filesize=12.0 K 2024-12-03T05:15:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/7d6b906bbcd241aca5dfdc8cab68c814 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814 2024-12-03T05:15:06,009 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814, entries=150, sequenceid=264, filesize=12.0 K 2024-12-03T05:15:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/6acc55d0e7a0459ab96a10b8ec893a27 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27 2024-12-03T05:15:06,019 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27, entries=150, sequenceid=264, filesize=12.0 K 2024-12-03T05:15:06,021 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fc274659350fd9dda12032ec9405be21 in 981ms, sequenceid=264, compaction requested=false 2024-12-03T05:15:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T05:15:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T05:15:06,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T05:15:06,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-12-03T05:15:06,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8180 sec 2024-12-03T05:15:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:06,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:15:06,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:06,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/602e68bffda8472b82b291b40a77f979 is 50, key is test_row_0/A:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:06,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741897_1073 (size=14741) 2024-12-03T05:15:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T05:15:06,371 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:06,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T05:15:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:06,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:06,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:06,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202966378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202966379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202966380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202966383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202966400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:06,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202966490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202966490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202966490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202966501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:06,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:06,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202966701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202966701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202966701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202966710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:06,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/602e68bffda8472b82b291b40a77f979 2024-12-03T05:15:06,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c10e799fbb7d42c8ac93f620247e0766 is 50, key is test_row_0/B:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:06,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741898_1074 (size=12301) 2024-12-03T05:15:06,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:06,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:06,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:07,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202967011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202967022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202967022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202967028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:07,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:07,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:07,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c10e799fbb7d42c8ac93f620247e0766 2024-12-03T05:15:07,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/94d0bf50b7d048229e64538ec6f03855 is 50, key is test_row_0/C:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:07,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741899_1075 (size=12301) 2024-12-03T05:15:07,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/94d0bf50b7d048229e64538ec6f03855 2024-12-03T05:15:07,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/602e68bffda8472b82b291b40a77f979 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979 2024-12-03T05:15:07,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979, entries=200, sequenceid=285, filesize=14.4 K 2024-12-03T05:15:07,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c10e799fbb7d42c8ac93f620247e0766 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766 2024-12-03T05:15:07,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T05:15:07,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/94d0bf50b7d048229e64538ec6f03855 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855 2024-12-03T05:15:07,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T05:15:07,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fc274659350fd9dda12032ec9405be21 in 947ms, sequenceid=285, compaction requested=true 2024-12-03T05:15:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:07,290 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:07,292 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:07,297 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:07,297 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:07,297 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,297 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/e3bcf2f053594eaea0cc57fca7dac1c9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=38.9 K 2024-12-03T05:15:07,300 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e3bcf2f053594eaea0cc57fca7dac1c9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:07,305 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e6901040014eeb850019f257540519, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733202903949 2024-12-03T05:15:07,305 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 602e68bffda8472b82b291b40a77f979, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:07,308 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:07,308 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:07,308 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,309 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bc32aa5cc4634779a1ee2e6d451a0e11, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.5 K 2024-12-03T05:15:07,311 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc32aa5cc4634779a1ee2e6d451a0e11, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:07,313 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6b906bbcd241aca5dfdc8cab68c814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733202903949 2024-12-03T05:15:07,314 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10e799fbb7d42c8ac93f620247e0766, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T05:15:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,325 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:07,343 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:07,343 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/221c50f80c5044ffb1bf64fa35967312 is 50, key is test_row_0/A:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fc2cb79c705f4d28a2e4a8b80136270a is 50, key is test_row_0/A:col10/1733202906381/Put/seqid=0 2024-12-03T05:15:07,359 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:07,360 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/b743320408154e48bb389e54ddfd7e4e is 50, key is test_row_0/B:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:07,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741901_1077 (size=12301) 2024-12-03T05:15:07,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741900_1076 (size=13017) 2024-12-03T05:15:07,413 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/221c50f80c5044ffb1bf64fa35967312 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/221c50f80c5044ffb1bf64fa35967312 2024-12-03T05:15:07,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:07,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741902_1078 (size=13017) 2024-12-03T05:15:07,434 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into 221c50f80c5044ffb1bf64fa35967312(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:07,435 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:07,435 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202907290; duration=0sec 2024-12-03T05:15:07,435 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:07,435 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:07,435 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:07,436 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:07,436 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:07,437 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:07,437 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6e9f649f996447bb897ce54e42c257be, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.5 K 2024-12-03T05:15:07,438 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9f649f996447bb897ce54e42c257be, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202903253 2024-12-03T05:15:07,439 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acc55d0e7a0459ab96a10b8ec893a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733202903949 2024-12-03T05:15:07,440 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d0bf50b7d048229e64538ec6f03855, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:07,454 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:07,455 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/da3503b1018649de8a1571aa9a5e8fa5 is 50, key is test_row_0/C:col10/1733202905213/Put/seqid=0 2024-12-03T05:15:07,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741903_1079 (size=13017) 2024-12-03T05:15:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:07,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202967529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202967531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202967532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202967535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202967541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202967640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202967660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fc2cb79c705f4d28a2e4a8b80136270a 2024-12-03T05:15:07,833 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/b743320408154e48bb389e54ddfd7e4e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b743320408154e48bb389e54ddfd7e4e 2024-12-03T05:15:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/83ef66caf09e48e699ffa8c41e670f22 is 50, key is test_row_0/B:col10/1733202906381/Put/seqid=0 2024-12-03T05:15:07,848 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into b743320408154e48bb389e54ddfd7e4e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:07,848 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:07,848 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202907291; duration=0sec 2024-12-03T05:15:07,849 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:07,849 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741904_1080 (size=12301) 2024-12-03T05:15:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202967852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202967896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:07,908 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/da3503b1018649de8a1571aa9a5e8fa5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/da3503b1018649de8a1571aa9a5e8fa5 2024-12-03T05:15:07,918 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into da3503b1018649de8a1571aa9a5e8fa5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:07,918 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:07,918 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202907291; duration=0sec 2024-12-03T05:15:07,918 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:07,918 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:08,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202968170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202968211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,254 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/83ef66caf09e48e699ffa8c41e670f22 2024-12-03T05:15:08,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e3060915052d48aa9270120c4c73d34c is 50, key is test_row_0/C:col10/1733202906381/Put/seqid=0 2024-12-03T05:15:08,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741905_1081 (size=12301) 2024-12-03T05:15:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:08,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202968551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,552 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202968561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,562 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202968561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,566 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,672 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e3060915052d48aa9270120c4c73d34c 2024-12-03T05:15:08,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fc2cb79c705f4d28a2e4a8b80136270a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a 2024-12-03T05:15:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202968682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,688 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T05:15:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/83ef66caf09e48e699ffa8c41e670f22 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22 2024-12-03T05:15:08,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T05:15:08,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/e3060915052d48aa9270120c4c73d34c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c 2024-12-03T05:15:08,716 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T05:15:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fc274659350fd9dda12032ec9405be21 in 1399ms, sequenceid=297, compaction requested=false 2024-12-03T05:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T05:15:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T05:15:08,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T05:15:08,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3530 sec 2024-12-03T05:15:08,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.3620 sec 2024-12-03T05:15:08,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202907508, max timestamp=1733202908735Num uniques 859; 2024-12-03T05:15:08,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:08,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202907508, max timestamp=1733202908735Num uniques 859; 2024-12-03T05:15:08,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:08,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-03T05:15:08,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:08,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:08,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:08,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:08,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:08,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:08,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/ee13a949aa4442e89cdca03eccc54f83 is 50, key is test_row_0/A:col10/1733202908735/Put/seqid=0 2024-12-03T05:15:08,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741906_1082 (size=12301) 2024-12-03T05:15:08,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/ee13a949aa4442e89cdca03eccc54f83 2024-12-03T05:15:08,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/19c34388c650439681270d242e7427c9 is 50, key is test_row_0/B:col10/1733202908735/Put/seqid=0 2024-12-03T05:15:08,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741907_1083 (size=12301) 2024-12-03T05:15:08,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/19c34388c650439681270d242e7427c9 2024-12-03T05:15:08,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/17f61550e3b94ea1b9757fed71655dfc is 50, key is test_row_0/C:col10/1733202908735/Put/seqid=0 2024-12-03T05:15:08,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741908_1084 (size=12301) 2024-12-03T05:15:08,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/17f61550e3b94ea1b9757fed71655dfc 2024-12-03T05:15:08,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/ee13a949aa4442e89cdca03eccc54f83 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83 2024-12-03T05:15:08,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83, entries=150, sequenceid=321, filesize=12.0 K 2024-12-03T05:15:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/19c34388c650439681270d242e7427c9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9 2024-12-03T05:15:08,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9, entries=150, sequenceid=321, filesize=12.0 K 2024-12-03T05:15:08,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/17f61550e3b94ea1b9757fed71655dfc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc 2024-12-03T05:15:08,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc, entries=150, sequenceid=321, filesize=12.0 K 2024-12-03T05:15:08,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=107.34 KB/109920 for fc274659350fd9dda12032ec9405be21 in 108ms, sequenceid=321, compaction requested=true 2024-12-03T05:15:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:08,885 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:08,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:08,886 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:08,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:08,887 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:08,887 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:08,887 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:08,887 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/221c50f80c5044ffb1bf64fa35967312, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.7 K 2024-12-03T05:15:08,888 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:08,888 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221c50f80c5044ffb1bf64fa35967312, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:08,888 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:08,888 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:08,888 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b743320408154e48bb389e54ddfd7e4e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.7 K 2024-12-03T05:15:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:08,888 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b743320408154e48bb389e54ddfd7e4e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:08,888 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2cb79c705f4d28a2e4a8b80136270a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733202906375 2024-12-03T05:15:08,889 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ef66caf09e48e699ffa8c41e670f22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733202906375 2024-12-03T05:15:08,889 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee13a949aa4442e89cdca03eccc54f83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202907508 2024-12-03T05:15:08,889 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c34388c650439681270d242e7427c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202907508 2024-12-03T05:15:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:08,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:08,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:08,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6b79443764564c6b9619e89662494813 is 50, key is test_row_0/A:col10/1733202908835/Put/seqid=0 2024-12-03T05:15:08,915 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:08,915 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/9a8c6ef5e5e4468cba3efaed76dabeda is 50, key is test_row_0/B:col10/1733202908735/Put/seqid=0 2024-12-03T05:15:08,927 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:08,927 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/05fd15393206476eb5d71071ff303fc5 is 50, key is test_row_0/A:col10/1733202908735/Put/seqid=0 2024-12-03T05:15:08,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741909_1085 (size=17181) 2024-12-03T05:15:08,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6b79443764564c6b9619e89662494813 2024-12-03T05:15:08,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202968979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:08,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741910_1086 (size=13119) 2024-12-03T05:15:09,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741911_1087 (size=13119) 2024-12-03T05:15:09,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/73447bca1dd441b1b727b3a27258bcab is 50, key is test_row_0/B:col10/1733202908835/Put/seqid=0 2024-12-03T05:15:09,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741912_1088 (size=12301) 2024-12-03T05:15:09,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/73447bca1dd441b1b727b3a27258bcab 2024-12-03T05:15:09,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/32630283035b4bbaa29d4739b8b41799 is 50, key is test_row_0/C:col10/1733202908835/Put/seqid=0 2024-12-03T05:15:09,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202969090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:09,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741913_1089 (size=12301) 2024-12-03T05:15:09,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/32630283035b4bbaa29d4739b8b41799 2024-12-03T05:15:09,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/6b79443764564c6b9619e89662494813 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813 2024-12-03T05:15:09,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813, entries=250, sequenceid=341, filesize=16.8 K 2024-12-03T05:15:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/73447bca1dd441b1b727b3a27258bcab as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab 2024-12-03T05:15:09,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T05:15:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/32630283035b4bbaa29d4739b8b41799 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799 2024-12-03T05:15:09,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T05:15:09,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fc274659350fd9dda12032ec9405be21 in 243ms, sequenceid=341, compaction requested=true 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:09,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:15:09,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:09,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fe6bc8233acb4d24b708e338889af692 is 50, key is test_row_0/A:col10/1733202908976/Put/seqid=0 2024-12-03T05:15:09,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741914_1090 (size=14741) 2024-12-03T05:15:09,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fe6bc8233acb4d24b708e338889af692 2024-12-03T05:15:09,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bb7384beac654dfa99076382e3ff50ba is 50, key is test_row_0/B:col10/1733202908976/Put/seqid=0 2024-12-03T05:15:09,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741915_1091 (size=12301) 2024-12-03T05:15:09,406 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/9a8c6ef5e5e4468cba3efaed76dabeda as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/9a8c6ef5e5e4468cba3efaed76dabeda 2024-12-03T05:15:09,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bb7384beac654dfa99076382e3ff50ba 2024-12-03T05:15:09,428 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/05fd15393206476eb5d71071ff303fc5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/05fd15393206476eb5d71071ff303fc5 2024-12-03T05:15:09,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f6bbd367f47b4a43beabb1019004728e is 50, key is test_row_0/C:col10/1733202908976/Put/seqid=0 2024-12-03T05:15:09,438 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into 9a8c6ef5e5e4468cba3efaed76dabeda(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:09,438 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:09,438 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202908886; duration=0sec 2024-12-03T05:15:09,438 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:15:09,438 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:09,438 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-03T05:15:09,444 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:09,444 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:09,445 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into 05fd15393206476eb5d71071ff303fc5(size=12.8 K), total size for store is 29.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:09,445 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202908885; duration=0sec 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:09,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:09,447 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:09,447 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/da3503b1018649de8a1571aa9a5e8fa5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=48.8 K 2024-12-03T05:15:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:09,447 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202969439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:09,449 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:09,449 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:09,449 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:09,449 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:09,456 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting da3503b1018649de8a1571aa9a5e8fa5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733202905213 2024-12-03T05:15:09,457 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e3060915052d48aa9270120c4c73d34c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733202906375 2024-12-03T05:15:09,458 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 17f61550e3b94ea1b9757fed71655dfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202907508 2024-12-03T05:15:09,458 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 32630283035b4bbaa29d4739b8b41799, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733202908835 2024-12-03T05:15:09,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741916_1092 (size=12301) 2024-12-03T05:15:09,485 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:09,485 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/c0f9a67fc5fe43859c6121a8abded980 is 50, key is test_row_0/C:col10/1733202908835/Put/seqid=0 2024-12-03T05:15:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741917_1093 (size=13153) 2024-12-03T05:15:09,541 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/c0f9a67fc5fe43859c6121a8abded980 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/c0f9a67fc5fe43859c6121a8abded980 2024-12-03T05:15:09,553 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into c0f9a67fc5fe43859c6121a8abded980(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:09,553 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:09,553 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=12, startTime=1733202909137; duration=0sec 2024-12-03T05:15:09,553 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:09,553 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202969563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202969700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:09,704 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202969770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:09,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f6bbd367f47b4a43beabb1019004728e 2024-12-03T05:15:09,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/fe6bc8233acb4d24b708e338889af692 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692 2024-12-03T05:15:09,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692, entries=200, sequenceid=354, filesize=14.4 K 2024-12-03T05:15:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/bb7384beac654dfa99076382e3ff50ba as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba 2024-12-03T05:15:09,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T05:15:09,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/f6bbd367f47b4a43beabb1019004728e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e 2024-12-03T05:15:09,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T05:15:09,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fc274659350fd9dda12032ec9405be21 in 614ms, sequenceid=354, compaction requested=true 2024-12-03T05:15:09,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:09,924 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:09,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:09,927 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:09,927 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:09,927 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:09,927 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/05fd15393206476eb5d71071ff303fc5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=44.0 K 2024-12-03T05:15:09,928 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 05fd15393206476eb5d71071ff303fc5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202907508 2024-12-03T05:15:09,928 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:09,929 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b79443764564c6b9619e89662494813, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733202908835 2024-12-03T05:15:09,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fe6bc8233acb4d24b708e338889af692, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733202908910 2024-12-03T05:15:09,931 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:09,931 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:09,931 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:09,931 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/9a8c6ef5e5e4468cba3efaed76dabeda, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.8 K 2024-12-03T05:15:09,932 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8c6ef5e5e4468cba3efaed76dabeda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733202907508 2024-12-03T05:15:09,933 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73447bca1dd441b1b727b3a27258bcab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733202908835 2024-12-03T05:15:09,934 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7384beac654dfa99076382e3ff50ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733202908910 2024-12-03T05:15:09,963 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:09,964 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c2a5e0f628be47fe9da5980c72f52ffe is 50, key is test_row_0/B:col10/1733202908976/Put/seqid=0 2024-12-03T05:15:09,968 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:09,969 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a6d97cbfb6f64dfea2b0248aeac11dea is 50, key is test_row_0/A:col10/1733202908976/Put/seqid=0 2024-12-03T05:15:10,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741918_1094 (size=13221) 2024-12-03T05:15:10,022 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c2a5e0f628be47fe9da5980c72f52ffe as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2a5e0f628be47fe9da5980c72f52ffe 2024-12-03T05:15:10,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741919_1095 (size=13221) 2024-12-03T05:15:10,031 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into c2a5e0f628be47fe9da5980c72f52ffe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:10,031 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202909925; duration=0sec 2024-12-03T05:15:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:10,033 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:10,033 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:10,033 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:10,033 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:10,043 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a6d97cbfb6f64dfea2b0248aeac11dea as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a6d97cbfb6f64dfea2b0248aeac11dea 2024-12-03T05:15:10,052 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into a6d97cbfb6f64dfea2b0248aeac11dea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:10,052 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:10,052 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202909924; duration=0sec 2024-12-03T05:15:10,053 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:10,053 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:10,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:15:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/344022fe0454428a8ab5ae699c5b39e9 is 50, key is test_row_0/A:col10/1733202910085/Put/seqid=0 2024-12-03T05:15:10,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741920_1096 (size=12301) 2024-12-03T05:15:10,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/344022fe0454428a8ab5ae699c5b39e9 2024-12-03T05:15:10,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3f61c8c36eb54325a12f7f59715d8852 is 50, key is test_row_0/B:col10/1733202910085/Put/seqid=0 2024-12-03T05:15:10,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741921_1097 (size=12301) 2024-12-03T05:15:10,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3f61c8c36eb54325a12f7f59715d8852 2024-12-03T05:15:10,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202970184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/19d7df162d464ebdbddff6cff6be84ad is 50, key is test_row_0/C:col10/1733202910085/Put/seqid=0 2024-12-03T05:15:10,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741922_1098 (size=12301) 2024-12-03T05:15:10,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/19d7df162d464ebdbddff6cff6be84ad 2024-12-03T05:15:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/344022fe0454428a8ab5ae699c5b39e9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9 2024-12-03T05:15:10,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9, entries=150, sequenceid=379, filesize=12.0 K 2024-12-03T05:15:10,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/3f61c8c36eb54325a12f7f59715d8852 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852 2024-12-03T05:15:10,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852, entries=150, sequenceid=379, filesize=12.0 K 2024-12-03T05:15:10,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/19d7df162d464ebdbddff6cff6be84ad as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad 2024-12-03T05:15:10,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202970290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad, entries=150, sequenceid=379, filesize=12.0 K 2024-12-03T05:15:10,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fc274659350fd9dda12032ec9405be21 in 216ms, sequenceid=379, compaction requested=true 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:10,304 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:10,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:10,306 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:10,306 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:10,306 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/c0f9a67fc5fe43859c6121a8abded980, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.9 K 2024-12-03T05:15:10,307 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f9a67fc5fe43859c6121a8abded980, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733202908835 2024-12-03T05:15:10,307 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f6bbd367f47b4a43beabb1019004728e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733202908910 2024-12-03T05:15:10,308 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d7df162d464ebdbddff6cff6be84ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733202909375 2024-12-03T05:15:10,326 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:10,327 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/2794c4c58af3492ca082b883bd1a420f is 50, key is test_row_0/C:col10/1733202910085/Put/seqid=0 2024-12-03T05:15:10,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741923_1099 (size=13255) 2024-12-03T05:15:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:10,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:10,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/40b92e9f61d74b0b945afce46b0c9742 is 50, key is test_row_0/A:col10/1733202910501/Put/seqid=0 2024-12-03T05:15:10,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741924_1100 (size=14741) 2024-12-03T05:15:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T05:15:10,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:10,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T05:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:10,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:10,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:10,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202970557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43998 deadline: 1733202970570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1733202970570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,572 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,572 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44074 deadline: 1733202970581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,582 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in fc274659350fd9dda12032ec9405be21 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:10,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202970671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:10,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:10,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:10,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:10,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,744 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/2794c4c58af3492ca082b883bd1a420f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2794c4c58af3492ca082b883bd1a420f 2024-12-03T05:15:10,753 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into 2794c4c58af3492ca082b883bd1a420f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:10,753 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:10,753 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=13, startTime=1733202910304; duration=0sec 2024-12-03T05:15:10,753 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:10,753 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:10,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:10,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:10,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202970880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:10,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/40b92e9f61d74b0b945afce46b0c9742 2024-12-03T05:15:10,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/a83517f6676a4c31a78b6ac063841073 is 50, key is test_row_0/B:col10/1733202910501/Put/seqid=0 2024-12-03T05:15:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741925_1101 (size=12301) 2024-12-03T05:15:10,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/a83517f6676a4c31a78b6ac063841073 2024-12-03T05:15:11,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/461a4fe7550e4d75811b327f0fedea7f is 50, key is test_row_0/C:col10/1733202910501/Put/seqid=0 2024-12-03T05:15:11,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741926_1102 (size=12301) 2024-12-03T05:15:11,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:11,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202971190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:11,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:11,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:11,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/461a4fe7550e4d75811b327f0fedea7f 2024-12-03T05:15:11,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/40b92e9f61d74b0b945afce46b0c9742 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742 2024-12-03T05:15:11,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742, entries=200, sequenceid=392, filesize=14.4 K 2024-12-03T05:15:11,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/a83517f6676a4c31a78b6ac063841073 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073 2024-12-03T05:15:11,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073, entries=150, sequenceid=392, filesize=12.0 K 2024-12-03T05:15:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/461a4fe7550e4d75811b327f0fedea7f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f 2024-12-03T05:15:11,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:11,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:11,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:11,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f, entries=150, sequenceid=392, filesize=12.0 K 2024-12-03T05:15:11,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fc274659350fd9dda12032ec9405be21 in 978ms, sequenceid=392, compaction requested=true 2024-12-03T05:15:11,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:11,481 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:11,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:11,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:11,482 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:11,482 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:11,483 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:11,483 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,483 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a6d97cbfb6f64dfea2b0248aeac11dea, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=39.3 K 2024-12-03T05:15:11,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:11,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:11,484 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,484 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2a5e0f628be47fe9da5980c72f52ffe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=36.9 K 2024-12-03T05:15:11,486 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d97cbfb6f64dfea2b0248aeac11dea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733202908910 2024-12-03T05:15:11,486 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a5e0f628be47fe9da5980c72f52ffe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733202908910 2024-12-03T05:15:11,486 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 344022fe0454428a8ab5ae699c5b39e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733202909375 2024-12-03T05:15:11,486 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f61c8c36eb54325a12f7f59715d8852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733202909375 2024-12-03T05:15:11,487 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40b92e9f61d74b0b945afce46b0c9742, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733202910125 2024-12-03T05:15:11,487 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a83517f6676a4c31a78b6ac063841073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733202910136 2024-12-03T05:15:11,505 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:11,506 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/454c1e1780c14a78be09891eb3fb6d5e is 50, key is test_row_0/A:col10/1733202910501/Put/seqid=0 2024-12-03T05:15:11,508 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:11,509 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c4e039c037d04c13a9b7f13a4119e6b7 is 50, key is test_row_0/B:col10/1733202910501/Put/seqid=0 2024-12-03T05:15:11,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741927_1103 (size=13323) 2024-12-03T05:15:11,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741928_1104 (size=13323) 2024-12-03T05:15:11,537 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/454c1e1780c14a78be09891eb3fb6d5e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/454c1e1780c14a78be09891eb3fb6d5e 2024-12-03T05:15:11,544 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into 454c1e1780c14a78be09891eb3fb6d5e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:11,544 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:11,544 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202911481; duration=0sec 2024-12-03T05:15:11,545 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:11,545 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:11,545 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:11,547 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:11,547 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:11,547 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:11,547 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:11,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T05:15:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:11,626 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/59816685f79d42c3a26a0f3613de9549 is 50, key is test_row_0/A:col10/1733202910556/Put/seqid=0 2024-12-03T05:15:11,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741929_1105 (size=12301) 2024-12-03T05:15:11,647 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/59816685f79d42c3a26a0f3613de9549 2024-12-03T05:15:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2b3051207d734babac4169648db9aaba is 50, key is test_row_0/B:col10/1733202910556/Put/seqid=0 2024-12-03T05:15:11,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741930_1106 (size=12301) 2024-12-03T05:15:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:11,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202971795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202971795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202971900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202971910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:11,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/c4e039c037d04c13a9b7f13a4119e6b7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c4e039c037d04c13a9b7f13a4119e6b7 2024-12-03T05:15:11,936 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into c4e039c037d04c13a9b7f13a4119e6b7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:11,937 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:11,937 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202911481; duration=0sec 2024-12-03T05:15:11,938 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:11,938 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:12,067 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2b3051207d734babac4169648db9aaba 2024-12-03T05:15:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/eed0e7fdef1c4bb089c206f46a06b240 is 50, key is test_row_0/C:col10/1733202910556/Put/seqid=0 2024-12-03T05:15:12,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202972113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741931_1107 (size=12301) 2024-12-03T05:15:12,124 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/eed0e7fdef1c4bb089c206f46a06b240 2024-12-03T05:15:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202972120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/59816685f79d42c3a26a0f3613de9549 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549 2024-12-03T05:15:12,144 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T05:15:12,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/2b3051207d734babac4169648db9aaba as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba 2024-12-03T05:15:12,157 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T05:15:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/eed0e7fdef1c4bb089c206f46a06b240 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240 2024-12-03T05:15:12,179 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T05:15:12,180 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for fc274659350fd9dda12032ec9405be21 in 555ms, sequenceid=413, compaction requested=true 2024-12-03T05:15:12,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:12,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:12,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T05:15:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T05:15:12,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T05:15:12,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-12-03T05:15:12,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6470 sec 2024-12-03T05:15:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:12,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T05:15:12,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:12,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:12,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:12,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:12,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:12,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:12,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/8d792ec436d34778a5a3e12001e2512a is 50, key is test_row_0/A:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:12,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741932_1108 (size=14741) 2024-12-03T05:15:12,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/8d792ec436d34778a5a3e12001e2512a 2024-12-03T05:15:12,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/274920eb909d457dbd03fa9d11d21621 is 50, key is test_row_0/B:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202972478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202972480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741933_1109 (size=12301) 2024-12-03T05:15:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202972590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202972590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T05:15:12,690 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:12,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-03T05:15:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:12,695 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:12,696 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:12,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202972800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202972802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:12,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:15:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:12,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:12,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/274920eb909d457dbd03fa9d11d21621 2024-12-03T05:15:12,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/d14ce804752743ccb4e95c3ccf24f852 is 50, key is test_row_0/C:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:12,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741934_1110 (size=12301) 2024-12-03T05:15:12,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/d14ce804752743ccb4e95c3ccf24f852 2024-12-03T05:15:12,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/8d792ec436d34778a5a3e12001e2512a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a 2024-12-03T05:15:12,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a, entries=200, sequenceid=429, filesize=14.4 K 2024-12-03T05:15:12,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/274920eb909d457dbd03fa9d11d21621 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621 2024-12-03T05:15:12,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T05:15:12,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/d14ce804752743ccb4e95c3ccf24f852 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852 2024-12-03T05:15:12,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852, entries=150, sequenceid=429, filesize=12.0 K 2024-12-03T05:15:12,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for fc274659350fd9dda12032ec9405be21 in 531ms, sequenceid=429, compaction requested=true 2024-12-03T05:15:12,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:12,955 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:12,955 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:12,956 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:12,956 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:12,956 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/A is initiating minor compaction (all files) 2024-12-03T05:15:12,956 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/B is initiating minor compaction (all files) 2024-12-03T05:15:12,956 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/A in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:12,956 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/B in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:12,956 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/454c1e1780c14a78be09891eb3fb6d5e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=39.4 K 2024-12-03T05:15:12,956 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c4e039c037d04c13a9b7f13a4119e6b7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=37.0 K 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454c1e1780c14a78be09891eb3fb6d5e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733202910136 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e039c037d04c13a9b7f13a4119e6b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733202910136 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59816685f79d42c3a26a0f3613de9549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733202910547 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3051207d734babac4169648db9aaba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733202910547 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d792ec436d34778a5a3e12001e2512a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733202911755 2024-12-03T05:15:12,957 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 274920eb909d457dbd03fa9d11d21621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733202911755 2024-12-03T05:15:12,977 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:12,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/c126a108d9314579bdf235d9dde4c9ba is 50, key is test_row_0/A:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:12,980 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#B#compaction#98 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:12,980 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/b19855d75b794b9db769ebf416846a05 is 50, key is test_row_0/B:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:12,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741935_1111 (size=13425) 2024-12-03T05:15:13,001 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/c126a108d9314579bdf235d9dde4c9ba as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c126a108d9314579bdf235d9dde4c9ba 2024-12-03T05:15:13,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T05:15:13,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:13,005 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:13,010 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/A of fc274659350fd9dda12032ec9405be21 into c126a108d9314579bdf235d9dde4c9ba(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:13,010 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:13,010 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/A, priority=13, startTime=1733202912955; duration=0sec 2024-12-03T05:15:13,010 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:13,010 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:13,010 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:13,013 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:13,013 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): fc274659350fd9dda12032ec9405be21/C is initiating minor compaction (all files) 2024-12-03T05:15:13,013 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc274659350fd9dda12032ec9405be21/C in TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:13,014 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2794c4c58af3492ca082b883bd1a420f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp, totalSize=49.0 K 2024-12-03T05:15:13,016 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2794c4c58af3492ca082b883bd1a420f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733202909375 2024-12-03T05:15:13,016 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461a4fe7550e4d75811b327f0fedea7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733202910136 2024-12-03T05:15:13,016 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed0e7fdef1c4bb089c206f46a06b240, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733202910547 2024-12-03T05:15:13,017 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14ce804752743ccb4e95c3ccf24f852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733202911755 2024-12-03T05:15:13,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741936_1112 (size=13425) 2024-12-03T05:15:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/d17b6041e9ee4208b800ea5fc98d477b is 50, key is test_row_0/A:col10/1733202912470/Put/seqid=0 2024-12-03T05:15:13,033 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc274659350fd9dda12032ec9405be21#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:13,033 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/106d952fbbaa4c9b8dd0d329749a539e is 50, key is test_row_0/C:col10/1733202912422/Put/seqid=0 2024-12-03T05:15:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741937_1113 (size=12301) 2024-12-03T05:15:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741938_1114 (size=13391) 2024-12-03T05:15:13,073 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/106d952fbbaa4c9b8dd0d329749a539e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/106d952fbbaa4c9b8dd0d329749a539e 2024-12-03T05:15:13,080 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc274659350fd9dda12032ec9405be21/C of fc274659350fd9dda12032ec9405be21 into 106d952fbbaa4c9b8dd0d329749a539e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:13,080 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:13,080 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/C, priority=12, startTime=1733202912955; duration=0sec 2024-12-03T05:15:13,080 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:13,080 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:13,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. as already flushing 2024-12-03T05:15:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202973136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202973137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202973251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202973251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:13,431 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/b19855d75b794b9db769ebf416846a05 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b19855d75b794b9db769ebf416846a05 2024-12-03T05:15:13,436 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc274659350fd9dda12032ec9405be21/B of fc274659350fd9dda12032ec9405be21 into b19855d75b794b9db769ebf416846a05(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:13,436 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:13,437 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., storeName=fc274659350fd9dda12032ec9405be21/B, priority=13, startTime=1733202912955; duration=0sec 2024-12-03T05:15:13,437 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:13,437 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:13,441 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/d17b6041e9ee4208b800ea5fc98d477b 2024-12-03T05:15:13,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/79bb0afebfbb4b6c91aff1fc950da3db is 50, key is test_row_0/B:col10/1733202912470/Put/seqid=0 2024-12-03T05:15:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202973461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202973461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741939_1115 (size=12301) 2024-12-03T05:15:13,470 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/79bb0afebfbb4b6c91aff1fc950da3db 2024-12-03T05:15:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/0b4a749ac53040f2a20a6b5a82b7a1bb is 50, key is test_row_0/C:col10/1733202912470/Put/seqid=0 2024-12-03T05:15:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741940_1116 (size=12301) 2024-12-03T05:15:13,625 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T05:15:13,625 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:13,626 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T05:15:13,626 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:13,628 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T05:15:13,628 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:13,630 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,630 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,630 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,631 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,631 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,631 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:13,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:13,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:13,633 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T05:15:13,634 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:13,634 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,634 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:13,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44016 deadline: 1733202973771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44060 deadline: 1733202973771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:13,892 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/0b4a749ac53040f2a20a6b5a82b7a1bb 2024-12-03T05:15:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/d17b6041e9ee4208b800ea5fc98d477b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/d17b6041e9ee4208b800ea5fc98d477b 2024-12-03T05:15:13,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/d17b6041e9ee4208b800ea5fc98d477b, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T05:15:13,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/79bb0afebfbb4b6c91aff1fc950da3db as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/79bb0afebfbb4b6c91aff1fc950da3db 2024-12-03T05:15:13,919 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/79bb0afebfbb4b6c91aff1fc950da3db, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T05:15:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/0b4a749ac53040f2a20a6b5a82b7a1bb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/0b4a749ac53040f2a20a6b5a82b7a1bb 2024-12-03T05:15:13,924 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/0b4a749ac53040f2a20a6b5a82b7a1bb, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T05:15:13,925 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fc274659350fd9dda12032ec9405be21 in 921ms, sequenceid=447, compaction requested=false 2024-12-03T05:15:13,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:13,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:13,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-03T05:15:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-03T05:15:13,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T05:15:13,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2300 sec 2024-12-03T05:15:13,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2380 sec 2024-12-03T05:15:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:14,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T05:15:14,283 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:14,283 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:14,283 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:14,283 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:14,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,284 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T05:15:14,284 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:14,284 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,284 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/04f1f085c0954c3ab0699b7c87aa7658 is 50, key is test_row_0/A:col10/1733202914281/Put/seqid=0 2024-12-03T05:15:14,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741941_1117 (size=12301) 2024-12-03T05:15:14,602 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T05:15:14,602 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:14,602 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,602 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,613 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T05:15:14,613 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:14,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,622 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T05:15:14,622 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:14,622 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,622 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/04f1f085c0954c3ab0699b7c87aa7658 2024-12-03T05:15:14,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/70b5bbbdb0d1469a8aa37ba85f0497ad is 50, key is test_row_0/B:col10/1733202914281/Put/seqid=0 2024-12-03T05:15:14,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741942_1118 (size=12301) 2024-12-03T05:15:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T05:15:14,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:14,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-03T05:15:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2974 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2881 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4151 rows 2024-12-03T05:15:14,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1392 2024-12-03T05:15:14,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4168 rows 2024-12-03T05:15:14,845 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:15:14,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:15:14,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:14,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:14,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:15:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:14,862 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202914862"}]},"ts":"1733202914862"} 2024-12-03T05:15:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:14,865 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:15:14,865 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:15:14,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:15:14,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, UNASSIGN}] 2024-12-03T05:15:14,875 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, UNASSIGN 2024-12-03T05:15:14,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fc274659350fd9dda12032ec9405be21, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:14,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, UNASSIGN because future has completed 2024-12-03T05:15:14,880 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:15:14,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:15,044 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:15,044 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:15:15,045 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing fc274659350fd9dda12032ec9405be21, disabling compactions & flushes 2024-12-03T05:15:15,045 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:15,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/70b5bbbdb0d1469a8aa37ba85f0497ad 2024-12-03T05:15:15,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7b6e2ebb2f8a44758aac7cffff65f7f5 is 50, key is test_row_0/C:col10/1733202914281/Put/seqid=0 2024-12-03T05:15:15,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741943_1119 (size=12301) 2024-12-03T05:15:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:15,482 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:15:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:15,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7b6e2ebb2f8a44758aac7cffff65f7f5 2024-12-03T05:15:15,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/04f1f085c0954c3ab0699b7c87aa7658 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/04f1f085c0954c3ab0699b7c87aa7658 2024-12-03T05:15:15,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/04f1f085c0954c3ab0699b7c87aa7658, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T05:15:15,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/70b5bbbdb0d1469a8aa37ba85f0497ad as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/70b5bbbdb0d1469a8aa37ba85f0497ad 2024-12-03T05:15:15,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/70b5bbbdb0d1469a8aa37ba85f0497ad, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T05:15:15,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/7b6e2ebb2f8a44758aac7cffff65f7f5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7b6e2ebb2f8a44758aac7cffff65f7f5 2024-12-03T05:15:15,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7b6e2ebb2f8a44758aac7cffff65f7f5, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T05:15:15,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for fc274659350fd9dda12032ec9405be21 in 1282ms, sequenceid=466, compaction requested=true 2024-12-03T05:15:15,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc274659350fd9dda12032ec9405be21: 2024-12-03T05:15:15,565 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:15,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:15,565 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:15,565 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:15,565 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:A 2024-12-03T05:15:15,565 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. after waiting 0 ms 2024-12-03T05:15:15,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:15,565 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:15,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:B, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:15,565 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:15,566 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:B 2024-12-03T05:15:15,566 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing fc274659350fd9dda12032ec9405be21 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T05:15:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc274659350fd9dda12032ec9405be21:C, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:15,566 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. because compaction request was cancelled 2024-12-03T05:15:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:15,566 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc274659350fd9dda12032ec9405be21:C 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=A 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=B 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc274659350fd9dda12032ec9405be21, store=C 2024-12-03T05:15:15,566 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:15,570 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a35ca46a4a2d446387b01a049f179d2e is 50, key is test_row_0/A:col10/1733202914621/Put/seqid=0 2024-12-03T05:15:15,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741944_1120 (size=9857) 2024-12-03T05:15:15,980 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a35ca46a4a2d446387b01a049f179d2e 2024-12-03T05:15:15,993 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e6184d17446a4396937f7a13d9be8ea1 is 50, key is test_row_0/B:col10/1733202914621/Put/seqid=0 2024-12-03T05:15:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:16,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741945_1121 (size=9857) 2024-12-03T05:15:16,409 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e6184d17446a4396937f7a13d9be8ea1 2024-12-03T05:15:16,421 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/fdc55b7d43f74c7f88020554d44a5e39 is 50, key is test_row_0/C:col10/1733202914621/Put/seqid=0 2024-12-03T05:15:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741946_1122 (size=9857) 2024-12-03T05:15:16,828 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/fdc55b7d43f74c7f88020554d44a5e39 2024-12-03T05:15:16,834 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/A/a35ca46a4a2d446387b01a049f179d2e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a35ca46a4a2d446387b01a049f179d2e 2024-12-03T05:15:16,839 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a35ca46a4a2d446387b01a049f179d2e, entries=100, sequenceid=472, filesize=9.6 K 2024-12-03T05:15:16,840 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/B/e6184d17446a4396937f7a13d9be8ea1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e6184d17446a4396937f7a13d9be8ea1 2024-12-03T05:15:16,847 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e6184d17446a4396937f7a13d9be8ea1, entries=100, sequenceid=472, filesize=9.6 K 2024-12-03T05:15:16,850 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/.tmp/C/fdc55b7d43f74c7f88020554d44a5e39 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/fdc55b7d43f74c7f88020554d44a5e39 2024-12-03T05:15:16,856 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/fdc55b7d43f74c7f88020554d44a5e39, entries=100, sequenceid=472, filesize=9.6 K 2024-12-03T05:15:16,857 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc274659350fd9dda12032ec9405be21 in 1292ms, sequenceid=472, compaction requested=true 2024-12-03T05:15:16,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/68e84851e4af43639475e64795472941, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a8c1d514b3124c0ebc33770cf95fed02, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2565bbae7c6488cafcda43da601139f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b497542d40b84663a75907174693f93c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b3c5353a5e4d477888cde436d040928f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/e3bcf2f053594eaea0cc57fca7dac1c9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/221c50f80c5044ffb1bf64fa35967312, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/05fd15393206476eb5d71071ff303fc5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a6d97cbfb6f64dfea2b0248aeac11dea, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/454c1e1780c14a78be09891eb3fb6d5e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a] to archive 2024-12-03T05:15:16,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:16,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/94a8d12b25564549b5e1c493e8daaf45 2024-12-03T05:15:16,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2861b8f887d47cfbdd34f61d2a42a72 2024-12-03T05:15:16,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/32628457e8084500bc1750533bc02b20 2024-12-03T05:15:16,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/68e84851e4af43639475e64795472941 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/68e84851e4af43639475e64795472941 2024-12-03T05:15:16,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/4a6c2436b6c04504ad9e7fe19c1784f6 2024-12-03T05:15:16,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a8c1d514b3124c0ebc33770cf95fed02 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a8c1d514b3124c0ebc33770cf95fed02 2024-12-03T05:15:16,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/da6748a8c1924a118c932d589487ae1a 2024-12-03T05:15:16,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/9e499224e5c649bfae33626a80bf4c6f 2024-12-03T05:15:16,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/009810ddc2d14378a8ca42c711a137c7 2024-12-03T05:15:16,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2565bbae7c6488cafcda43da601139f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b2565bbae7c6488cafcda43da601139f 2024-12-03T05:15:16,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/34f4c6ef09e84b3c95197bf138aade31 2024-12-03T05:15:16,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b497542d40b84663a75907174693f93c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b497542d40b84663a75907174693f93c 2024-12-03T05:15:16,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/2c7f2defd2f44869aab91ba9b4a10de2 2024-12-03T05:15:16,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/bd5824c1ca3a41b09fc64096454e3d62 2024-12-03T05:15:16,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6e94cb013fa74284a30a5b1c3eb886f5 2024-12-03T05:15:16,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b3c5353a5e4d477888cde436d040928f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/b3c5353a5e4d477888cde436d040928f 2024-12-03T05:15:16,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c4d26604315141998d075ee25f167913 2024-12-03T05:15:16,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/e3bcf2f053594eaea0cc57fca7dac1c9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/e3bcf2f053594eaea0cc57fca7dac1c9 2024-12-03T05:15:16,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/0bc15588410e4616a8f3d54eed4008c8 2024-12-03T05:15:16,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/54e6901040014eeb850019f257540519 2024-12-03T05:15:16,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/602e68bffda8472b82b291b40a77f979 2024-12-03T05:15:16,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/221c50f80c5044ffb1bf64fa35967312 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/221c50f80c5044ffb1bf64fa35967312 2024-12-03T05:15:16,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fc2cb79c705f4d28a2e4a8b80136270a 2024-12-03T05:15:16,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/05fd15393206476eb5d71071ff303fc5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/05fd15393206476eb5d71071ff303fc5 2024-12-03T05:15:16,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/ee13a949aa4442e89cdca03eccc54f83 2024-12-03T05:15:16,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/6b79443764564c6b9619e89662494813 2024-12-03T05:15:16,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/fe6bc8233acb4d24b708e338889af692 2024-12-03T05:15:16,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a6d97cbfb6f64dfea2b0248aeac11dea to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a6d97cbfb6f64dfea2b0248aeac11dea 2024-12-03T05:15:16,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/344022fe0454428a8ab5ae699c5b39e9 2024-12-03T05:15:16,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/40b92e9f61d74b0b945afce46b0c9742 2024-12-03T05:15:16,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/454c1e1780c14a78be09891eb3fb6d5e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/454c1e1780c14a78be09891eb3fb6d5e 2024-12-03T05:15:16,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/59816685f79d42c3a26a0f3613de9549 2024-12-03T05:15:16,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/8d792ec436d34778a5a3e12001e2512a 2024-12-03T05:15:16,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2b395529f8145cdb5ac32f5dd5396eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bada4a6d4290434ab2361452d4dd66e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/fbd0c8b48dbd4864b46d36eb52f6a9b6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7ca38243c9cb451697565566d42f19eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/24508790506c4121af9f864029ca77aa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bc32aa5cc4634779a1ee2e6d451a0e11, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b743320408154e48bb389e54ddfd7e4e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/9a8c6ef5e5e4468cba3efaed76dabeda, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2a5e0f628be47fe9da5980c72f52ffe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c4e039c037d04c13a9b7f13a4119e6b7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621] to archive 2024-12-03T05:15:16,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:16,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/245dbd437f2d4adfbebd7548e0f5a808 2024-12-03T05:15:16,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/faf460640f254851ab021bb1037d0185 2024-12-03T05:15:16,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2b395529f8145cdb5ac32f5dd5396eb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2b395529f8145cdb5ac32f5dd5396eb 2024-12-03T05:15:16,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/f38e51b10e28414e8f2aa557770e6153 2024-12-03T05:15:16,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/0c986c6745504a70b5a7dfbcc625af3a 2024-12-03T05:15:16,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bada4a6d4290434ab2361452d4dd66e7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bada4a6d4290434ab2361452d4dd66e7 2024-12-03T05:15:16,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/6e785e7171d04803803d497ac2c7f21f 2024-12-03T05:15:16,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2f413b57b82545b98aa121bc086ea552 2024-12-03T05:15:16,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/fbd0c8b48dbd4864b46d36eb52f6a9b6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/fbd0c8b48dbd4864b46d36eb52f6a9b6 2024-12-03T05:15:16,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/98d2d9dc961f49688aab01807cdb1714 2024-12-03T05:15:16,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c7303a5089f54ffca32417a8338e4ada 2024-12-03T05:15:16,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7ca38243c9cb451697565566d42f19eb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7ca38243c9cb451697565566d42f19eb 2024-12-03T05:15:16,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3759ac7afaf4484690ae07ab02e8cbce 2024-12-03T05:15:16,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72f5f2c0d57c44558b152b9a176b7279 2024-12-03T05:15:16,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/24508790506c4121af9f864029ca77aa to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/24508790506c4121af9f864029ca77aa 2024-12-03T05:15:16,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/72ae72889ff5421e9fe92a9e25471d86 2024-12-03T05:15:16,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7484251936ec435cbd9d39e65cb6fbde 2024-12-03T05:15:16,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bc32aa5cc4634779a1ee2e6d451a0e11 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bc32aa5cc4634779a1ee2e6d451a0e11 2024-12-03T05:15:16,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e28dfa8ef35845cb937c66f68318395b 2024-12-03T05:15:16,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/7d6b906bbcd241aca5dfdc8cab68c814 2024-12-03T05:15:16,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b743320408154e48bb389e54ddfd7e4e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b743320408154e48bb389e54ddfd7e4e 2024-12-03T05:15:16,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c10e799fbb7d42c8ac93f620247e0766 2024-12-03T05:15:16,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/83ef66caf09e48e699ffa8c41e670f22 2024-12-03T05:15:16,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/9a8c6ef5e5e4468cba3efaed76dabeda to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/9a8c6ef5e5e4468cba3efaed76dabeda 2024-12-03T05:15:16,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/19c34388c650439681270d242e7427c9 2024-12-03T05:15:16,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/73447bca1dd441b1b727b3a27258bcab 2024-12-03T05:15:16,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2a5e0f628be47fe9da5980c72f52ffe to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c2a5e0f628be47fe9da5980c72f52ffe 2024-12-03T05:15:16,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/bb7384beac654dfa99076382e3ff50ba 2024-12-03T05:15:16,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/3f61c8c36eb54325a12f7f59715d8852 2024-12-03T05:15:16,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c4e039c037d04c13a9b7f13a4119e6b7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/c4e039c037d04c13a9b7f13a4119e6b7 2024-12-03T05:15:16,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/a83517f6676a4c31a78b6ac063841073 2024-12-03T05:15:16,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/2b3051207d734babac4169648db9aaba 2024-12-03T05:15:16,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/274920eb909d457dbd03fa9d11d21621 2024-12-03T05:15:16,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/91a0683ac5bc4a7d9b748b1b9ef0e56f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b307e01dd37a4069bd796dd88d5c5724, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b7da38bd058d45c3b4999e6f5b9e052e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b12d34f28c3c474aa9421ee078d46cf0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a56efaab58e843889a68a54db7461574, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6e9f649f996447bb897ce54e42c257be, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/da3503b1018649de8a1571aa9a5e8fa5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/c0f9a67fc5fe43859c6121a8abded980, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2794c4c58af3492ca082b883bd1a420f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852] to archive 2024-12-03T05:15:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:16,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b3c76fb861754c8192a0d897683cc6ad 2024-12-03T05:15:16,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/3a17f830918c4d99be243ec7cb5ed8d3 2024-12-03T05:15:16,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/91a0683ac5bc4a7d9b748b1b9ef0e56f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/91a0683ac5bc4a7d9b748b1b9ef0e56f 2024-12-03T05:15:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2b4d53df0e1a4196bdeff0163edb4855 2024-12-03T05:15:16,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a2de992963f44c66b80fc98ee91e5ed2 2024-12-03T05:15:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b307e01dd37a4069bd796dd88d5c5724 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b307e01dd37a4069bd796dd88d5c5724 2024-12-03T05:15:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/5ccee404b02e471189f629ecb19b1403 2024-12-03T05:15:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e02ca0cc0bd04428848fa56b0fe68a8d 2024-12-03T05:15:16,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b7da38bd058d45c3b4999e6f5b9e052e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b7da38bd058d45c3b4999e6f5b9e052e 2024-12-03T05:15:16,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/352f3bfd91894a448abed927fd5fbb2b 2024-12-03T05:15:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f3bcd1f71824489199bf5abdc366cfa8 2024-12-03T05:15:16,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b12d34f28c3c474aa9421ee078d46cf0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/b12d34f28c3c474aa9421ee078d46cf0 2024-12-03T05:15:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/649dbcc4d48b40d0aeafc64d01fa8389 2024-12-03T05:15:16,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7d26d39a27fd4477a4a2647e3ac673ef 2024-12-03T05:15:17,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a56efaab58e843889a68a54db7461574 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/a56efaab58e843889a68a54db7461574 2024-12-03T05:15:17,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/254fe6f6b2994cc5bdd0e7ce1de9f3df 2024-12-03T05:15:17,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/55874f9cba4b409581d9bd1ef7b33bf3 2024-12-03T05:15:17,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6e9f649f996447bb897ce54e42c257be to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6e9f649f996447bb897ce54e42c257be 2024-12-03T05:15:17,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/16739f1984814fbe986c1acce58cfb68 2024-12-03T05:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:17,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/6acc55d0e7a0459ab96a10b8ec893a27 2024-12-03T05:15:17,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/da3503b1018649de8a1571aa9a5e8fa5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/da3503b1018649de8a1571aa9a5e8fa5 2024-12-03T05:15:17,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/94d0bf50b7d048229e64538ec6f03855 2024-12-03T05:15:17,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/e3060915052d48aa9270120c4c73d34c 2024-12-03T05:15:17,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/17f61550e3b94ea1b9757fed71655dfc 2024-12-03T05:15:17,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/c0f9a67fc5fe43859c6121a8abded980 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/c0f9a67fc5fe43859c6121a8abded980 2024-12-03T05:15:17,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/32630283035b4bbaa29d4739b8b41799 2024-12-03T05:15:17,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/f6bbd367f47b4a43beabb1019004728e 2024-12-03T05:15:17,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2794c4c58af3492ca082b883bd1a420f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/2794c4c58af3492ca082b883bd1a420f 2024-12-03T05:15:17,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/19d7df162d464ebdbddff6cff6be84ad 2024-12-03T05:15:17,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/461a4fe7550e4d75811b327f0fedea7f 2024-12-03T05:15:17,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/eed0e7fdef1c4bb089c206f46a06b240 2024-12-03T05:15:17,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/d14ce804752743ccb4e95c3ccf24f852 2024-12-03T05:15:17,058 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=1 2024-12-03T05:15:17,059 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. 2024-12-03T05:15:17,059 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for fc274659350fd9dda12032ec9405be21: Waiting for close lock at 1733202915045Running coprocessor pre-close hooks at 1733202915045Disabling compacts and flushes for region at 1733202915045Disabling writes for close at 1733202915565 (+520 ms)Obtaining lock to block concurrent updates at 1733202915566 (+1 ms)Preparing flush snapshotting stores in fc274659350fd9dda12032ec9405be21 at 1733202915566Finished memstore snapshotting TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733202915566Flushing stores of TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21. at 1733202915567 (+1 ms)Flushing fc274659350fd9dda12032ec9405be21/A: creating writer at 1733202915567Flushing fc274659350fd9dda12032ec9405be21/A: appending metadata at 1733202915570 (+3 ms)Flushing fc274659350fd9dda12032ec9405be21/A: closing flushed file at 1733202915570Flushing fc274659350fd9dda12032ec9405be21/B: creating writer at 1733202915988 (+418 ms)Flushing fc274659350fd9dda12032ec9405be21/B: appending metadata at 1733202915993 (+5 ms)Flushing fc274659350fd9dda12032ec9405be21/B: closing flushed file at 1733202915993Flushing fc274659350fd9dda12032ec9405be21/C: creating writer at 1733202916416 (+423 ms)Flushing fc274659350fd9dda12032ec9405be21/C: appending metadata at 1733202916420 (+4 ms)Flushing fc274659350fd9dda12032ec9405be21/C: closing flushed file at 1733202916420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57c2b3c0: reopening flushed file at 1733202916833 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@692957ce: reopening flushed file at 1733202916839 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f188449: reopening flushed file at 1733202916847 (+8 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc274659350fd9dda12032ec9405be21 in 1292ms, sequenceid=472, compaction requested=true at 1733202916857 (+10 ms)Writing region close event to WAL at 1733202917054 (+197 ms)Running coprocessor post-close hooks at 1733202917059 (+5 ms)Closed at 1733202917059 2024-12-03T05:15:17,062 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:17,063 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fc274659350fd9dda12032ec9405be21, regionState=CLOSED 2024-12-03T05:15:17,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:17,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T05:15:17,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure fc274659350fd9dda12032ec9405be21, server=e3c6570475b6,37863,1733202887099 in 2.1850 sec 2024-12-03T05:15:17,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-03T05:15:17,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc274659350fd9dda12032ec9405be21, UNASSIGN in 2.1950 sec 2024-12-03T05:15:17,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T05:15:17,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2040 sec 2024-12-03T05:15:17,075 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202917075"}]},"ts":"1733202917075"} 2024-12-03T05:15:17,077 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:15:17,077 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:15:17,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2240 sec 2024-12-03T05:15:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T05:15:19,041 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:19,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:15:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,049 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T05:15:19,051 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:19,059 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/recovered.edits] 2024-12-03T05:15:19,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/04f1f085c0954c3ab0699b7c87aa7658 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/04f1f085c0954c3ab0699b7c87aa7658 2024-12-03T05:15:19,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a35ca46a4a2d446387b01a049f179d2e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/a35ca46a4a2d446387b01a049f179d2e 2024-12-03T05:15:19,064 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c126a108d9314579bdf235d9dde4c9ba to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/c126a108d9314579bdf235d9dde4c9ba 2024-12-03T05:15:19,065 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/d17b6041e9ee4208b800ea5fc98d477b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/A/d17b6041e9ee4208b800ea5fc98d477b 2024-12-03T05:15:19,067 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/70b5bbbdb0d1469a8aa37ba85f0497ad to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/70b5bbbdb0d1469a8aa37ba85f0497ad 2024-12-03T05:15:19,068 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/79bb0afebfbb4b6c91aff1fc950da3db to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/79bb0afebfbb4b6c91aff1fc950da3db 2024-12-03T05:15:19,069 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b19855d75b794b9db769ebf416846a05 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/b19855d75b794b9db769ebf416846a05 2024-12-03T05:15:19,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e6184d17446a4396937f7a13d9be8ea1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/B/e6184d17446a4396937f7a13d9be8ea1 2024-12-03T05:15:19,072 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/0b4a749ac53040f2a20a6b5a82b7a1bb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/0b4a749ac53040f2a20a6b5a82b7a1bb 2024-12-03T05:15:19,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/106d952fbbaa4c9b8dd0d329749a539e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/106d952fbbaa4c9b8dd0d329749a539e 2024-12-03T05:15:19,074 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7b6e2ebb2f8a44758aac7cffff65f7f5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/7b6e2ebb2f8a44758aac7cffff65f7f5 2024-12-03T05:15:19,076 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/fdc55b7d43f74c7f88020554d44a5e39 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/C/fdc55b7d43f74c7f88020554d44a5e39 2024-12-03T05:15:19,078 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/recovered.edits/475.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21/recovered.edits/475.seqid 2024-12-03T05:15:19,079 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/fc274659350fd9dda12032ec9405be21 2024-12-03T05:15:19,079 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:15:19,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T05:15:19,095 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:15:19,129 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:15:19,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:15:19,131 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202919131"}]},"ts":"9223372036854775807"} 2024-12-03T05:15:19,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:15:19,135 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc274659350fd9dda12032ec9405be21, NAME => 'TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:15:19,136 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:15:19,136 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202919136"}]},"ts":"9223372036854775807"} 2024-12-03T05:15:19,139 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:15:19,140 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-12-03T05:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T05:15:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:15:19,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:19,171 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1264088159_22 at /127.0.0.1:58228 [Waiting for operation #339] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e3c6570475b6:37863-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-443467210_22 at /127.0.0.1:50170 [Waiting for operation #343] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=719 (was 639) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3835 (was 3883) 2024-12-03T05:15:19,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=719, ProcessCount=11, AvailableMemoryMB=3835 2024-12-03T05:15:19,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:15:19,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:15:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:19,186 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:15:19,186 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:19,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-03T05:15:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:19,188 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:15:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741947_1123 (size=1061) 2024-12-03T05:15:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:19,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:15:19,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741948_1124 (size=53) 2024-12-03T05:15:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:20,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:15:20,010 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b7e2b9d2e05056c58426b072bf89a451, disabling compactions & flushes 2024-12-03T05:15:20,010 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,010 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,010 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. after waiting 0 ms 2024-12-03T05:15:20,010 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,010 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,010 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7e2b9d2e05056c58426b072bf89a451: Waiting for close lock at 1733202920009Disabling compacts and flushes for region at 1733202920009Disabling writes for close at 1733202920010 (+1 ms)Writing region close event to WAL at 1733202920010Closed at 1733202920010 2024-12-03T05:15:20,011 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:15:20,012 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202920011"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202920011"}]},"ts":"1733202920011"} 2024-12-03T05:15:20,014 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:15:20,015 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:15:20,016 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202920015"}]},"ts":"1733202920015"} 2024-12-03T05:15:20,018 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:15:20,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, ASSIGN}] 2024-12-03T05:15:20,020 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, ASSIGN 2024-12-03T05:15:20,021 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:15:20,172 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:20,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, ASSIGN because future has completed 2024-12-03T05:15:20,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:20,340 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,340 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:15:20,340 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,340 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:15:20,340 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,341 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,345 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,347 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:20,348 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName A 2024-12-03T05:15:20,348 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:20,352 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:20,352 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,353 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:20,354 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName B 2024-12-03T05:15:20,354 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:20,355 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:20,355 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,356 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:20,356 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName C 2024-12-03T05:15:20,356 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:20,357 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:20,357 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,358 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,358 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,359 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,360 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,360 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,363 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:15:20,365 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,370 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:15:20,371 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened b7e2b9d2e05056c58426b072bf89a451; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75033335, jitterRate=0.11808381974697113}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:15:20,371 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:20,371 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for b7e2b9d2e05056c58426b072bf89a451: Running coprocessor pre-open hook at 1733202920341Writing region info on filesystem at 1733202920341Initializing all the Stores at 1733202920345 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202920345Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202920345Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202920345Cleaning up temporary data from old regions at 1733202920360 (+15 ms)Running coprocessor post-open hooks at 1733202920371 (+11 ms)Region opened successfully at 1733202920371 2024-12-03T05:15:20,373 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., pid=30, masterSystemTime=1733202920330 2024-12-03T05:15:20,375 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,375 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:20,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:20,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:20,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T05:15:20,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 in 204 msec 2024-12-03T05:15:20,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T05:15:20,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, ASSIGN in 367 msec 2024-12-03T05:15:20,391 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:15:20,391 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202920391"}]},"ts":"1733202920391"} 2024-12-03T05:15:20,393 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:15:20,394 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:15:20,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2100 sec 2024-12-03T05:15:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T05:15:21,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:21,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,338 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb71261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,339 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:21,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:21,344 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45945b10 2024-12-03T05:15:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:15:21,346 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:15:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:15:21,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:15:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:21,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741949_1125 (size=1097) 2024-12-03T05:15:21,405 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T05:15:21,405 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T05:15:21,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:15:21,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE}] 2024-12-03T05:15:21,417 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE 2024-12-03T05:15:21,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:21,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE because future has completed 2024-12-03T05:15:21,420 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:15:21,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:21,573 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,573 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:15:21,574 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing b7e2b9d2e05056c58426b072bf89a451, disabling compactions & flushes 2024-12-03T05:15:21,574 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,574 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,574 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. after waiting 0 ms 2024-12-03T05:15:21,574 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,579 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:15:21,580 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,580 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for b7e2b9d2e05056c58426b072bf89a451: Waiting for close lock at 1733202921574Running coprocessor pre-close hooks at 1733202921574Disabling compacts and flushes for region at 1733202921574Disabling writes for close at 1733202921574Writing region close event to WAL at 1733202921574Running coprocessor post-close hooks at 1733202921580 (+6 ms)Closed at 1733202921580 2024-12-03T05:15:21,581 WARN [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: b7e2b9d2e05056c58426b072bf89a451 to self. 2024-12-03T05:15:21,583 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,584 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=CLOSED 2024-12-03T05:15:21,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:21,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-03T05:15:21,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 in 167 msec 2024-12-03T05:15:21,590 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE; state=CLOSED, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=true 2024-12-03T05:15:21,741 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:21,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE because future has completed 2024-12-03T05:15:21,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:21,899 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,899 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:15:21,900 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,900 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:15:21,900 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,900 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,903 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,904 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:21,909 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName A 2024-12-03T05:15:21,911 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:21,912 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:21,912 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,913 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:21,913 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName B 2024-12-03T05:15:21,913 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:21,913 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:21,914 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,914 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:21,914 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7e2b9d2e05056c58426b072bf89a451 columnFamilyName C 2024-12-03T05:15:21,914 DEBUG [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:21,915 INFO [StoreOpener-b7e2b9d2e05056c58426b072bf89a451-1 {}] regionserver.HStore(327): Store=b7e2b9d2e05056c58426b072bf89a451/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:21,915 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,915 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,916 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,917 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,918 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,918 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,919 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:15:21,920 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,921 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b7e2b9d2e05056c58426b072bf89a451; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72582745, jitterRate=0.08156718313694}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:15:21,921 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:21,923 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b7e2b9d2e05056c58426b072bf89a451: Running coprocessor pre-open hook at 1733202921900Writing region info on filesystem at 1733202921900Initializing all the Stores at 1733202921901 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202921901Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202921901Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202921901Cleaning up temporary data from old regions at 1733202921918 (+17 ms)Running coprocessor post-open hooks at 1733202921921 (+3 ms)Region opened successfully at 1733202921923 (+2 ms) 2024-12-03T05:15:21,924 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., pid=35, masterSystemTime=1733202921895 2024-12-03T05:15:21,927 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,927 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:21,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=OPEN, openSeqNum=5, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:21,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:21,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-03T05:15:21,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 in 189 msec 2024-12-03T05:15:21,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T05:15:21,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, REOPEN/MOVE in 519 msec 2024-12-03T05:15:21,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T05:15:21,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-12-03T05:15:21,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 594 msec 2024-12-03T05:15:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T05:15:21,954 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:21,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c333d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,962 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,969 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a35b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f33bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,977 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67caca49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638f174a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4b27a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:21,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:21,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e16309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:21,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c700350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:22,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:22,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d88bea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:22,006 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685c174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:22,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:22,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed9c8f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:22,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:22,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f2083a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:22,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:22,020 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c495221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:22,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35883207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:22,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:22,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26fc9d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:22,033 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:22,037 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,049 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-03T05:15:22,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:22,056 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:22,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,063 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,063 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,063 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,065 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,066 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,068 DEBUG [Thread-637 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,069 DEBUG [Thread-637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,086 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:22,088 DEBUG [Thread-635 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:22,088 DEBUG [Thread-635 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:22,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:15:22,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:15:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:22,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dead705e6c8f4d36bef3c23f5562f65d_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202922126/Put/seqid=0 2024-12-03T05:15:22,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202922249, max timestamp=1733202922262Num uniques 859; 2024-12-03T05:15:22,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:22,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202922249, max timestamp=1733202922262Num uniques 859; 2024-12-03T05:15:22,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:22,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202922249, max timestamp=1733202922262Num uniques 859; 2024-12-03T05:15:22,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:22,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741950_1126 (size=12154) 2024-12-03T05:15:22,341 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:22,349 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dead705e6c8f4d36bef3c23f5562f65d_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dead705e6c8f4d36bef3c23f5562f65d_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:22,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/01a27813bf4d46c69d8b5b6bdf169cfc, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:22,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/01a27813bf4d46c69d8b5b6bdf169cfc is 175, key is test_row_0/A:col10/1733202922126/Put/seqid=0 2024-12-03T05:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:22,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202982353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741951_1127 (size=30955) 2024-12-03T05:15:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202982357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/01a27813bf4d46c69d8b5b6bdf169cfc 2024-12-03T05:15:22,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202982378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202982390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202982394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/469d3d08263f46f1958795187c92bf1c is 50, key is test_row_0/B:col10/1733202922126/Put/seqid=0 2024-12-03T05:15:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741952_1128 (size=12001) 2024-12-03T05:15:22,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/469d3d08263f46f1958795187c92bf1c 2024-12-03T05:15:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202982501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202982513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:22,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:22,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202982521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/bef1de9b3f824f85b3fe3897865ea326 is 50, key is test_row_0/C:col10/1733202922126/Put/seqid=0 2024-12-03T05:15:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202982531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202982531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741953_1129 (size=12001) 2024-12-03T05:15:22,676 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:22,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:22,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202982721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202982730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202982751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202982751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202982751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:22,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:22,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/bef1de9b3f824f85b3fe3897865ea326 2024-12-03T05:15:23,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/01a27813bf4d46c69d8b5b6bdf169cfc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc 2024-12-03T05:15:23,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc, entries=150, sequenceid=14, filesize=30.2 K 2024-12-03T05:15:23,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/469d3d08263f46f1958795187c92bf1c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c 2024-12-03T05:15:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T05:15:23,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/bef1de9b3f824f85b3fe3897865ea326 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326 2024-12-03T05:15:23,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T05:15:23,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b7e2b9d2e05056c58426b072bf89a451 in 920ms, sequenceid=14, compaction requested=false 2024-12-03T05:15:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:23,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733202922347, max timestamp=1733202923082Num uniques 1018; 2024-12-03T05:15:23,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:15:23,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733202922347, max timestamp=1733202923089Num uniques 1018; 2024-12-03T05:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:23,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=78.27 KB heapSize=218.64 KB 2024-12-03T05:15:23,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733202922347, max timestamp=1733202923082Num uniques 1018; 2024-12-03T05:15:23,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:15:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a78c71ad66e342d6954a7a4810ce6e24_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202923100/Put/seqid=0 2024-12-03T05:15:23,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741954_1130 (size=21918) 2024-12-03T05:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:23,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a78c71ad66e342d6954a7a4810ce6e24_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a78c71ad66e342d6954a7a4810ce6e24_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:23,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/10d59a11ed2842bcaaf50b2d978d0d65, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:23,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/10d59a11ed2842bcaaf50b2d978d0d65 is 175, key is test_row_0/A:col10/1733202923100/Put/seqid=0 2024-12-03T05:15:23,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741955_1131 (size=65323) 2024-12-03T05:15:23,270 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/10d59a11ed2842bcaaf50b2d978d0d65 2024-12-03T05:15:23,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733202923188, max timestamp=1733202923266Num uniques 859; 2024-12-03T05:15:23,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-03T05:15:23,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733202923188, max timestamp=1733202923266Num uniques 859; 2024-12-03T05:15:23,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-03T05:15:23,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733202923188, max timestamp=1733202923266Num uniques 859; 2024-12-03T05:15:23,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-03T05:15:23,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a19d3fd80a747cfb351cf4ac40421e7 is 50, key is test_row_0/B:col10/1733202923100/Put/seqid=0 2024-12-03T05:15:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741956_1132 (size=12001) 2024-12-03T05:15:23,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202983321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a19d3fd80a747cfb351cf4ac40421e7 2024-12-03T05:15:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202983321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202983328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:23,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202983343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202983353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ccb3c61f13584585a25a5f56700b6bf8 is 50, key is test_row_0/C:col10/1733202923100/Put/seqid=0 2024-12-03T05:15:23,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741957_1133 (size=12001) 2024-12-03T05:15:23,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ccb3c61f13584585a25a5f56700b6bf8 2024-12-03T05:15:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202983451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202983452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202983452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/10d59a11ed2842bcaaf50b2d978d0d65 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65 2024-12-03T05:15:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202983481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202983490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:23,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65, entries=350, sequenceid=55, filesize=63.8 K 2024-12-03T05:15:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a19d3fd80a747cfb351cf4ac40421e7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7 2024-12-03T05:15:23,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7, entries=150, sequenceid=55, filesize=11.7 K 2024-12-03T05:15:23,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ccb3c61f13584585a25a5f56700b6bf8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8 2024-12-03T05:15:23,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8, entries=150, sequenceid=55, filesize=11.7 K 2024-12-03T05:15:23,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.11 KB/128112, currentSize=154.31 KB/158010 for b7e2b9d2e05056c58426b072bf89a451 in 515ms, sequenceid=55, compaction requested=false 2024-12-03T05:15:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:23,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T05:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:23,674 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=154.31 KB heapSize=421.31 KB 2024-12-03T05:15:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:23,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202983698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203210d0634a4b643fe87c1e266f8df28e9_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202923339/Put/seqid=0 2024-12-03T05:15:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202983705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202983711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202983720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202983714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741958_1134 (size=12154) 2024-12-03T05:15:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202983821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202983831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202983840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202984031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202984031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202984041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202984041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202984053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:24,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203210d0634a4b643fe87c1e266f8df28e9_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203210d0634a4b643fe87c1e266f8df28e9_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/8803896f340d487d872e36245f1cf5fe, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/8803896f340d487d872e36245f1cf5fe is 175, key is test_row_0/A:col10/1733202923339/Put/seqid=0 2024-12-03T05:15:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:24,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741959_1135 (size=30955) 2024-12-03T05:15:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202984364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202984366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202984372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202984551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202984570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,640 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/8803896f340d487d872e36245f1cf5fe 2024-12-03T05:15:24,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/61a0550150dc4a4dacf65141f27e40e9 is 50, key is test_row_0/B:col10/1733202923339/Put/seqid=0 2024-12-03T05:15:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741960_1136 (size=12001) 2024-12-03T05:15:24,764 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/61a0550150dc4a4dacf65141f27e40e9 2024-12-03T05:15:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d0cd4a45bbad426d885561bf81973b57 is 50, key is test_row_0/C:col10/1733202923339/Put/seqid=0 2024-12-03T05:15:24,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741961_1137 (size=12001) 2024-12-03T05:15:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202984882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202984882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202984893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,247 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d0cd4a45bbad426d885561bf81973b57 2024-12-03T05:15:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/8803896f340d487d872e36245f1cf5fe as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe 2024-12-03T05:15:25,270 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe, entries=150, sequenceid=90, filesize=30.2 K 2024-12-03T05:15:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/61a0550150dc4a4dacf65141f27e40e9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9 2024-12-03T05:15:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T05:15:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d0cd4a45bbad426d885561bf81973b57 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57 2024-12-03T05:15:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,314 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T05:15:25,315 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~421.27 KB/431376, currentSize=33.54 KB/34350 for b7e2b9d2e05056c58426b072bf89a451 in 1641ms, sequenceid=90, compaction requested=true 2024-12-03T05:15:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-03T05:15:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-03T05:15:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T05:15:25,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2690 sec 2024-12-03T05:15:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.2770 sec 2024-12-03T05:15:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:25,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d09462ddf0304c0bbb9d1073a3263a9b_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:25,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741962_1138 (size=14594) 2024-12-03T05:15:25,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733202925711, max timestamp=1733202925762Num uniques 859; 2024-12-03T05:15:25,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:15:25,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733202925711, max timestamp=1733202925762Num uniques 859; 2024-12-03T05:15:25,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:15:25,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733202925711, max timestamp=1733202925762Num uniques 859; 2024-12-03T05:15:25,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:15:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202985852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202985862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202985890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,898 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202985901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,906 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202985911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,916 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202985971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202985984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:26,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:26,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d09462ddf0304c0bbb9d1073a3263a9b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d09462ddf0304c0bbb9d1073a3263a9b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:26,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d76774d05444414eaf6f26afbcaf9bac, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:26,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d76774d05444414eaf6f26afbcaf9bac is 175, key is test_row_0/A:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202986180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:26,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741963_1139 (size=39549) 2024-12-03T05:15:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202986200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T05:15:26,220 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:26,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T05:15:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:26,227 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:26,228 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:26,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:26,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202986492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202986514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:26,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:26,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,612 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d76774d05444414eaf6f26afbcaf9bac 2024-12-03T05:15:26,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/da425da4b7ea46a0a9f934545e7a4f72 is 50, key is test_row_0/B:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:26,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741964_1140 (size=12001) 2024-12-03T05:15:26,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:26,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:26,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:26,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202987001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:27,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202987031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:27,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/da425da4b7ea46a0a9f934545e7a4f72 2024-12-03T05:15:27,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e534823b05c48b79bb75cda81bdd5b4 is 50, key is test_row_0/C:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:27,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741965_1141 (size=12001) 2024-12-03T05:15:27,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e534823b05c48b79bb75cda81bdd5b4 2024-12-03T05:15:27,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d76774d05444414eaf6f26afbcaf9bac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac 2024-12-03T05:15:27,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac, entries=200, sequenceid=100, filesize=38.6 K 2024-12-03T05:15:27,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/da425da4b7ea46a0a9f934545e7a4f72 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72 2024-12-03T05:15:27,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:27,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:27,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:27,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T05:15:27,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e534823b05c48b79bb75cda81bdd5b4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4 2024-12-03T05:15:27,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T05:15:27,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b7e2b9d2e05056c58426b072bf89a451 in 1580ms, sequenceid=100, compaction requested=true 2024-12-03T05:15:27,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:27,211 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:27,211 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:27,212 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:27,212 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:27,212 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:27,213 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,213 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:27,213 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,213 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=162.9 K 2024-12-03T05:15:27,213 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=46.9 K 2024-12-03T05:15:27,213 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,213 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac] 2024-12-03T05:15:27,214 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01a27813bf4d46c69d8b5b6bdf169cfc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202922082 2024-12-03T05:15:27,214 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 469d3d08263f46f1958795187c92bf1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202922082 2024-12-03T05:15:27,214 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d59a11ed2842bcaaf50b2d978d0d65, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202922347 2024-12-03T05:15:27,215 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a19d3fd80a747cfb351cf4ac40421e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202923089 2024-12-03T05:15:27,215 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8803896f340d487d872e36245f1cf5fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202923311 2024-12-03T05:15:27,215 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a0550150dc4a4dacf65141f27e40e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202923311 2024-12-03T05:15:27,215 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76774d05444414eaf6f26afbcaf9bac, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202923701 2024-12-03T05:15:27,215 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting da425da4b7ea46a0a9f934545e7a4f72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202923707 2024-12-03T05:15:27,235 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:27,241 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120393dac3bff8e741458f8f431d86010e85_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:27,247 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120393dac3bff8e741458f8f431d86010e85_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:27,247 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393dac3bff8e741458f8f431d86010e85_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:27,251 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:27,252 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/88b0aa9b8d2b4fe3b1d107b982aae764 is 50, key is test_row_0/B:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741966_1142 (size=4469) 2024-12-03T05:15:27,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741967_1143 (size=12139) 2024-12-03T05:15:27,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T05:15:27,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:27,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ce30099f6d5241db892ba3f22b6a6131_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202925821/Put/seqid=0 2024-12-03T05:15:27,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741968_1144 (size=12304) 2024-12-03T05:15:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,382 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ce30099f6d5241db892ba3f22b6a6131_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce30099f6d5241db892ba3f22b6a6131_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/62886b0d171c44b798112f4c8a9bb989, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/62886b0d171c44b798112f4c8a9bb989 is 175, key is test_row_0/A:col10/1733202925821/Put/seqid=0 2024-12-03T05:15:27,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741969_1145 (size=31105) 2024-12-03T05:15:27,391 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/62886b0d171c44b798112f4c8a9bb989 2024-12-03T05:15:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a20d3b5a3b6465984c14bbf382fff44 is 50, key is test_row_0/B:col10/1733202925821/Put/seqid=0 2024-12-03T05:15:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741970_1146 (size=12151) 2024-12-03T05:15:27,406 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a20d3b5a3b6465984c14bbf382fff44 2024-12-03T05:15:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/80ee10a3d10e4adcb3c95581d623237a is 50, key is test_row_0/C:col10/1733202925821/Put/seqid=0 2024-12-03T05:15:27,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741971_1147 (size=12151) 2024-12-03T05:15:27,431 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/80ee10a3d10e4adcb3c95581d623237a 2024-12-03T05:15:27,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/62886b0d171c44b798112f4c8a9bb989 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989 2024-12-03T05:15:27,452 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989, entries=150, sequenceid=135, filesize=30.4 K 2024-12-03T05:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/4a20d3b5a3b6465984c14bbf382fff44 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44 2024-12-03T05:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44, entries=150, sequenceid=135, filesize=11.9 K 2024-12-03T05:15:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/80ee10a3d10e4adcb3c95581d623237a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a 2024-12-03T05:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,475 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a, entries=150, sequenceid=135, filesize=11.9 K 2024-12-03T05:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,476 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for b7e2b9d2e05056c58426b072bf89a451 in 153ms, sequenceid=135, compaction requested=true 2024-12-03T05:15:27,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:27,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T05:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T05:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T05:15:27,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-12-03T05:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.2610 sec 2024-12-03T05:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,665 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#122 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,666 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/c857001305b948a09e9059d6df19060d is 175, key is test_row_0/A:col10/1733202925624/Put/seqid=0 2024-12-03T05:15:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,686 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/88b0aa9b8d2b4fe3b1d107b982aae764 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/88b0aa9b8d2b4fe3b1d107b982aae764 2024-12-03T05:15:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,698 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 88b0aa9b8d2b4fe3b1d107b982aae764(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:27,698 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:27,698 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=12, startTime=1733202927211; duration=0sec 2024-12-03T05:15:27,698 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:27,698 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:27,698 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:15:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60155 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:15:27,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:27,703 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,703 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=58.7 K 2024-12-03T05:15:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,705 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bef1de9b3f824f85b3fe3897865ea326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733202922082 2024-12-03T05:15:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,706 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb3c61f13584585a25a5f56700b6bf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733202923089 2024-12-03T05:15:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,707 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd4a45bbad426d885561bf81973b57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733202923311 2024-12-03T05:15:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,708 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e534823b05c48b79bb75cda81bdd5b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202923707 2024-12-03T05:15:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,709 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ee10a3d10e4adcb3c95581d623237a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733202925821 2024-12-03T05:15:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741972_1148 (size=31093) 2024-12-03T05:15:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,744 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#127 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,744 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/7b89ca4224794349acd10db946b94b66 is 50, key is test_row_0/C:col10/1733202925821/Put/seqid=0 2024-12-03T05:15:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741973_1149 (size=12323) 2024-12-03T05:15:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed19a09e99f74ecc99ec1d0c1e6e5b48_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,157 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/c857001305b948a09e9059d6df19060d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d 2024-12-03T05:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,183 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into c857001305b948a09e9059d6df19060d(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:28,183 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:28,183 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=12, startTime=1733202927210; duration=0sec 2024-12-03T05:15:28,184 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:28,184 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:28,184 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/7b89ca4224794349acd10db946b94b66 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b89ca4224794349acd10db946b94b66 2024-12-03T05:15:28,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202928136, max timestamp=1733202928171Num uniques 859; 2024-12-03T05:15:28,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202928136, max timestamp=1733202928171Num uniques 859; 2024-12-03T05:15:28,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:28,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:28,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741974_1150 (size=22268) 2024-12-03T05:15:28,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202928136, max timestamp=1733202928171Num uniques 859; 2024-12-03T05:15:28,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:28,199 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into 7b89ca4224794349acd10db946b94b66(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:28,199 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:28,199 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=11, startTime=1733202927211; duration=0sec 2024-12-03T05:15:28,199 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:28,199 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:28,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202988240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202988240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202988241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202988244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202988244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202988360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202988361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202988362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202988362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202988362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T05:15:28,370 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:28,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T05:15:28,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:28,374 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:28,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:28,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:15:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202988573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202988573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202988573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202988573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202988576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:28,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed19a09e99f74ecc99ec1d0c1e6e5b48_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed19a09e99f74ecc99ec1d0c1e6e5b48_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:28,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/47ce868ec1724b2a88364d2b15f3fb7f, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:28,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/47ce868ec1724b2a88364d2b15f3fb7f is 175, key is test_row_0/A:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741975_1151 (size=65673) 2024-12-03T05:15:28,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/47ce868ec1724b2a88364d2b15f3fb7f 2024-12-03T05:15:28,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/98a9a34093b04e3482f462f1eaa71747 is 50, key is test_row_0/B:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:28,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741976_1152 (size=12151) 2024-12-03T05:15:28,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:15:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:28,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:15:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:28,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202988891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202988891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202988891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202988891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202988892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:28,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:15:28,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:28,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:28,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:29,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/98a9a34093b04e3482f462f1eaa71747 2024-12-03T05:15:29,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0777b87058194aa89a569c15771d2cee is 50, key is test_row_0/C:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:29,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741977_1153 (size=12151) 2024-12-03T05:15:29,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0777b87058194aa89a569c15771d2cee 2024-12-03T05:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/47ce868ec1724b2a88364d2b15f3fb7f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f 2024-12-03T05:15:29,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f, entries=350, sequenceid=146, filesize=64.1 K 2024-12-03T05:15:29,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/98a9a34093b04e3482f462f1eaa71747 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747 2024-12-03T05:15:29,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747, entries=150, sequenceid=146, filesize=11.9 K 2024-12-03T05:15:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0777b87058194aa89a569c15771d2cee as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee 2024-12-03T05:15:29,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee, entries=150, sequenceid=146, filesize=11.9 K 2024-12-03T05:15:29,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b7e2b9d2e05056c58426b072bf89a451 in 1062ms, sequenceid=146, compaction requested=true 2024-12-03T05:15:29,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:29,148 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:29,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:29,148 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:29,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T05:15:29,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:29,149 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T05:15:29,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:29,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:29,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:29,149 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:29,150 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:29,150 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:29,150 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:29,150 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:29,150 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=124.9 K 2024-12-03T05:15:29,150 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/88b0aa9b8d2b4fe3b1d107b982aae764, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=35.6 K 2024-12-03T05:15:29,150 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:29,150 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f] 2024-12-03T05:15:29,151 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c857001305b948a09e9059d6df19060d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202923707 2024-12-03T05:15:29,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b0aa9b8d2b4fe3b1d107b982aae764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733202923707 2024-12-03T05:15:29,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a20d3b5a3b6465984c14bbf382fff44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733202925821 2024-12-03T05:15:29,151 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62886b0d171c44b798112f4c8a9bb989, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733202925821 2024-12-03T05:15:29,152 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a9a34093b04e3482f462f1eaa71747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733202928060 2024-12-03T05:15:29,152 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47ce868ec1724b2a88364d2b15f3fb7f, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733202928057 2024-12-03T05:15:29,174 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:29,175 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/d47cf308260c4591b03c1e462efaa2e6 is 50, key is test_row_0/B:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:29,186 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:29,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120300a6909122fa43d49b93e6c936f2cc59_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202928241/Put/seqid=0 2024-12-03T05:15:29,232 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037eae244b13c741299ddd469fc0254a68_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:29,234 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037eae244b13c741299ddd469fc0254a68_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:29,234 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037eae244b13c741299ddd469fc0254a68_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:29,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741980_1156 (size=4469) 2024-12-03T05:15:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741979_1155 (size=12304) 2024-12-03T05:15:29,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741978_1154 (size=12391) 2024-12-03T05:15:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:29,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:29,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202989413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202989415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202989458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202989458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202989459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202989560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202989570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202989571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202989571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,669 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:29,670 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4ce15a074604406a85802ce69ffd892d is 175, key is test_row_0/A:col10/1733202928060/Put/seqid=0 2024-12-03T05:15:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:29,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120300a6909122fa43d49b93e6c936f2cc59_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120300a6909122fa43d49b93e6c936f2cc59_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:29,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/adfb8e4281cf4b96b272ecea1383f6d2, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/adfb8e4281cf4b96b272ecea1383f6d2 is 175, key is test_row_0/A:col10/1733202928241/Put/seqid=0 2024-12-03T05:15:29,693 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/d47cf308260c4591b03c1e462efaa2e6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d47cf308260c4591b03c1e462efaa2e6 2024-12-03T05:15:29,701 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into d47cf308260c4591b03c1e462efaa2e6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:29,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:29,701 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202929148; duration=0sec 2024-12-03T05:15:29,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:29,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:29,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:15:29,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:15:29,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:15:29,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. because compaction request was cancelled 2024-12-03T05:15:29,702 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:29,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741981_1157 (size=31345) 2024-12-03T05:15:29,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741982_1158 (size=31105) 2024-12-03T05:15:29,722 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/adfb8e4281cf4b96b272ecea1383f6d2 2024-12-03T05:15:29,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/1651e51235c843e0907dd4c539360874 is 50, key is test_row_0/B:col10/1733202928241/Put/seqid=0 2024-12-03T05:15:29,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202989771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741983_1159 (size=12151) 2024-12-03T05:15:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202989781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202989781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202989781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:29,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/1651e51235c843e0907dd4c539360874 2024-12-03T05:15:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b425c63d5ef94486b8b26ecc7313e087 is 50, key is test_row_0/C:col10/1733202928241/Put/seqid=0 2024-12-03T05:15:29,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741984_1160 (size=12151) 2024-12-03T05:15:29,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b425c63d5ef94486b8b26ecc7313e087 2024-12-03T05:15:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/adfb8e4281cf4b96b272ecea1383f6d2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2 2024-12-03T05:15:29,907 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2, entries=150, sequenceid=185, filesize=30.4 K 2024-12-03T05:15:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/1651e51235c843e0907dd4c539360874 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874 2024-12-03T05:15:29,926 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T05:15:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b425c63d5ef94486b8b26ecc7313e087 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087 2024-12-03T05:15:29,949 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T05:15:29,951 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for b7e2b9d2e05056c58426b072bf89a451 in 801ms, sequenceid=185, compaction requested=true 2024-12-03T05:15:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T05:15:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T05:15:29,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T05:15:29,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5790 sec 2024-12-03T05:15:29,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5890 sec 2024-12-03T05:15:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:30,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:30,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120350648a02068c4cdd94e19fbdaad17c71_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,123 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4ce15a074604406a85802ce69ffd892d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d 2024-12-03T05:15:30,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741985_1161 (size=14794) 2024-12-03T05:15:30,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:30,136 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 4ce15a074604406a85802ce69ffd892d(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:30,136 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:30,136 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202929148; duration=0sec 2024-12-03T05:15:30,136 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:30,136 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:30,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120350648a02068c4cdd94e19fbdaad17c71_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350648a02068c4cdd94e19fbdaad17c71_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:30,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/db070c21a6204621b0248940fbb66ac9, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/db070c21a6204621b0248940fbb66ac9 is 175, key is test_row_0/A:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202930130, max timestamp=1733202930147Num uniques 859; 2024-12-03T05:15:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202930130, max timestamp=1733202930147Num uniques 859; 2024-12-03T05:15:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T05:15:30,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T05:15:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202990136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202930130, max timestamp=1733202930147Num uniques 859; 2024-12-03T05:15:30,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T05:15:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741986_1162 (size=39749) 2024-12-03T05:15:30,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202990212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202990215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202990219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202990261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202990331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202990331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202990332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202990472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202990481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,493 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202990541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202990541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T05:15:30,545 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:30,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T05:15:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:30,549 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:30,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202990546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:30,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:30,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/db070c21a6204621b0248940fbb66ac9 2024-12-03T05:15:30,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/25ee3ba66bd54975acd3a1e4a65c189f is 50, key is test_row_0/B:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741987_1163 (size=12151) 2024-12-03T05:15:30,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/25ee3ba66bd54975acd3a1e4a65c189f 2024-12-03T05:15:30,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0e72600ffe9a4658a5ee8f0c109944f7 is 50, key is test_row_0/C:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741988_1164 (size=12151) 2024-12-03T05:15:30,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0e72600ffe9a4658a5ee8f0c109944f7 2024-12-03T05:15:30,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/db070c21a6204621b0248940fbb66ac9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9 2024-12-03T05:15:30,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9, entries=200, sequenceid=197, filesize=38.8 K 2024-12-03T05:15:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/25ee3ba66bd54975acd3a1e4a65c189f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f 2024-12-03T05:15:30,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f, entries=150, sequenceid=197, filesize=11.9 K 2024-12-03T05:15:30,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/0e72600ffe9a4658a5ee8f0c109944f7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7 2024-12-03T05:15:30,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7, entries=150, sequenceid=197, filesize=11.9 K 2024-12-03T05:15:30,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.28 KB/129315 for b7e2b9d2e05056c58426b072bf89a451 in 647ms, sequenceid=197, compaction requested=true 2024-12-03T05:15:30,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:30,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:30,735 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:30,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:30,735 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:30,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:30,737 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:30,737 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:30,737 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:30,737 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:30,737 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,737 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,737 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=99.8 K 2024-12-03T05:15:30,737 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d47cf308260c4591b03c1e462efaa2e6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=35.8 K 2024-12-03T05:15:30,737 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,737 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9] 2024-12-03T05:15:30,738 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce15a074604406a85802ce69ffd892d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733202928060 2024-12-03T05:15:30,739 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting adfb8e4281cf4b96b272ecea1383f6d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733202928210 2024-12-03T05:15:30,739 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d47cf308260c4591b03c1e462efaa2e6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733202928060 2024-12-03T05:15:30,740 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting db070c21a6204621b0248940fbb66ac9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929408 2024-12-03T05:15:30,740 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 1651e51235c843e0907dd4c539360874, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733202928210 2024-12-03T05:15:30,741 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ee3ba66bd54975acd3a1e4a65c189f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929415 2024-12-03T05:15:30,759 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,774 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203732afbed4a674cd895dcb1be877745bd_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,776 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203732afbed4a674cd895dcb1be877745bd_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,776 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203732afbed4a674cd895dcb1be877745bd_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,777 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#140 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:30,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/9f030801052c48ddbcd50e8881936a3b is 50, key is test_row_0/B:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:30,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741989_1165 (size=12493) 2024-12-03T05:15:30,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:30,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741990_1166 (size=4469) 2024-12-03T05:15:30,823 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/9f030801052c48ddbcd50e8881936a3b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/9f030801052c48ddbcd50e8881936a3b 2024-12-03T05:15:30,824 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#139 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:30,825 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/1a0a4717e8cb4f49837ed498695897ec is 175, key is test_row_0/A:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:30,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203368f6b3bd5104798ad6b42adaf599f3e_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202930205/Put/seqid=0 2024-12-03T05:15:30,869 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 9f030801052c48ddbcd50e8881936a3b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:30,869 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:30,869 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202930735; duration=0sec 2024-12-03T05:15:30,869 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:30,869 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:30,869 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202990866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741991_1167 (size=31447) 2024-12-03T05:15:30,878 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48776 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:30,878 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:30,878 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,879 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b89ca4224794349acd10db946b94b66, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=47.6 K 2024-12-03T05:15:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:30,881 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b89ca4224794349acd10db946b94b66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733202925821 2024-12-03T05:15:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741992_1168 (size=12304) 2024-12-03T05:15:30,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202990870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202990872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:30,887 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0777b87058194aa89a569c15771d2cee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733202928060 2024-12-03T05:15:30,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202990877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,887 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b425c63d5ef94486b8b26ecc7313e087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733202928210 2024-12-03T05:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,889 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e72600ffe9a4658a5ee8f0c109944f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929415 2024-12-03T05:15:30,900 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/1a0a4717e8cb4f49837ed498695897ec as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec 2024-12-03T05:15:30,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203368f6b3bd5104798ad6b42adaf599f3e_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203368f6b3bd5104798ad6b42adaf599f3e_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:30,905 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/a4abc63fa53147c9b3c8f71236c18342, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:30,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/a4abc63fa53147c9b3c8f71236c18342 is 175, key is test_row_0/A:col10/1733202930205/Put/seqid=0 2024-12-03T05:15:30,926 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 1a0a4717e8cb4f49837ed498695897ec(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:30,926 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:30,926 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202930735; duration=0sec 2024-12-03T05:15:30,926 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:30,926 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:30,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741993_1169 (size=31105) 2024-12-03T05:15:30,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=44.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/a4abc63fa53147c9b3c8f71236c18342 2024-12-03T05:15:30,946 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:30,946 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/3adcfe3d26834e73a71e2886e32c33c4 is 50, key is test_row_0/C:col10/1733202929415/Put/seqid=0 2024-12-03T05:15:30,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/0537cbc084904a17acb2b5cd88f799ef is 50, key is test_row_0/B:col10/1733202930205/Put/seqid=0 2024-12-03T05:15:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202990984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741994_1170 (size=12459) 2024-12-03T05:15:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741995_1171 (size=12151) 2024-12-03T05:15:31,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202991192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202991210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202991210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202991210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,400 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/3adcfe3d26834e73a71e2886e32c33c4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3adcfe3d26834e73a71e2886e32c33c4 2024-12-03T05:15:31,408 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into 3adcfe3d26834e73a71e2886e32c33c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:31,408 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:31,408 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=12, startTime=1733202930736; duration=0sec 2024-12-03T05:15:31,408 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:31,408 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:31,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/0537cbc084904a17acb2b5cd88f799ef 2024-12-03T05:15:31,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b5e6cf43e38e42b2ac74087cf94f8d87 is 50, key is test_row_0/C:col10/1733202930205/Put/seqid=0 2024-12-03T05:15:31,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741996_1172 (size=12151) 2024-12-03T05:15:31,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b5e6cf43e38e42b2ac74087cf94f8d87 2024-12-03T05:15:31,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/a4abc63fa53147c9b3c8f71236c18342 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342 2024-12-03T05:15:31,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342, entries=150, sequenceid=235, filesize=30.4 K 2024-12-03T05:15:31,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/0537cbc084904a17acb2b5cd88f799ef as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef 2024-12-03T05:15:31,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef, entries=150, sequenceid=235, filesize=11.9 K 2024-12-03T05:15:31,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/b5e6cf43e38e42b2ac74087cf94f8d87 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87 2024-12-03T05:15:31,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87, entries=150, sequenceid=235, filesize=11.9 K 2024-12-03T05:15:31,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136185, heapSize ~382.20 KB/391368, currentSize=46.96 KB/48090 for b7e2b9d2e05056c58426b072bf89a451 in 675ms, sequenceid=235, compaction requested=false 2024-12-03T05:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:31,503 INFO [master/e3c6570475b6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T05:15:31,503 INFO [master/e3c6570475b6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T05:15:31,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:31,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120365cf5e8df59e4a0ea631e4ca213be84b_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:31,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741997_1173 (size=14794) 2024-12-03T05:15:31,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202991578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202991578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202931531, max timestamp=1733202931583Num uniques 859; 2024-12-03T05:15:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202991581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202931531, max timestamp=1733202931583Num uniques 859; 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:31,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202931531, max timestamp=1733202931583Num uniques 859; 2024-12-03T05:15:31,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202991691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202991694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202991698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:31,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202991701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202991811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202991811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202991828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:31,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202991913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:31,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:31,964 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120365cf5e8df59e4a0ea631e4ca213be84b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120365cf5e8df59e4a0ea631e4ca213be84b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:31,965 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/85df70946b4e457baf227e0d1ef469da, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:31,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/85df70946b4e457baf227e0d1ef469da is 175, key is test_row_0/A:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:31,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741998_1174 (size=39749) 2024-12-03T05:15:31,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/85df70946b4e457baf227e0d1ef469da 2024-12-03T05:15:32,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/2a3749b75f0547e3a5a60ce220570229 is 50, key is test_row_0/B:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202992023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202992023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741999_1175 (size=12151) 2024-12-03T05:15:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202992045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202992231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202992332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202992332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202992371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/2a3749b75f0547e3a5a60ce220570229 2024-12-03T05:15:32,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e105cdcd8404e7aa60f327d4e27438b is 50, key is test_row_0/C:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:32,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:32,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:32,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:32,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742000_1176 (size=12151) 2024-12-03T05:15:32,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e105cdcd8404e7aa60f327d4e27438b 2024-12-03T05:15:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202992511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,515 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/85df70946b4e457baf227e0d1ef469da as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da 2024-12-03T05:15:32,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da, entries=200, sequenceid=249, filesize=38.8 K 2024-12-03T05:15:32,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/2a3749b75f0547e3a5a60ce220570229 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229 2024-12-03T05:15:32,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T05:15:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/2e105cdcd8404e7aa60f327d4e27438b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b 2024-12-03T05:15:32,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T05:15:32,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b7e2b9d2e05056c58426b072bf89a451 in 1097ms, sequenceid=249, compaction requested=true 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:32,600 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:32,600 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:32,615 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:32,615 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:32,615 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:32,615 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,615 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=99.9 K 2024-12-03T05:15:32,615 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,615 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da] 2024-12-03T05:15:32,616 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a0a4717e8cb4f49837ed498695897ec, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929415 2024-12-03T05:15:32,616 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:32,616 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,617 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/9f030801052c48ddbcd50e8881936a3b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=35.9 K 2024-12-03T05:15:32,617 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4abc63fa53147c9b3c8f71236c18342, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733202930200 2024-12-03T05:15:32,618 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85df70946b4e457baf227e0d1ef469da, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930864 2024-12-03T05:15:32,620 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f030801052c48ddbcd50e8881936a3b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929415 2024-12-03T05:15:32,629 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0537cbc084904a17acb2b5cd88f799ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733202930200 2024-12-03T05:15:32,631 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3749b75f0547e3a5a60ce220570229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930865 2024-12-03T05:15:32,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T05:15:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,649 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:32,665 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:32,708 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:32,709 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/d4d1e435a80d423d9bc20519702fbb88 is 50, key is test_row_0/B:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:32,710 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412032bb97b1ac61e4aa1b9cc5f59e4b200dc_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:32,712 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412032bb97b1ac61e4aa1b9cc5f59e4b200dc_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:32,712 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032bb97b1ac61e4aa1b9cc5f59e4b200dc_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:32,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120391097e4048f54cd3b4c010f5c2bb0d69_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202931696/Put/seqid=0 2024-12-03T05:15:32,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742001_1177 (size=12595) 2024-12-03T05:15:32,779 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/d4d1e435a80d423d9bc20519702fbb88 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d4d1e435a80d423d9bc20519702fbb88 2024-12-03T05:15:32,800 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into d4d1e435a80d423d9bc20519702fbb88(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:32,800 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:32,800 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202932600; duration=0sec 2024-12-03T05:15:32,800 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:32,800 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:32,800 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:32,802 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742002_1178 (size=4469) 2024-12-03T05:15:32,803 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:32,803 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:32,804 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3adcfe3d26834e73a71e2886e32c33c4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=35.9 K 2024-12-03T05:15:32,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742003_1179 (size=12454) 2024-12-03T05:15:32,811 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adcfe3d26834e73a71e2886e32c33c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733202929415 2024-12-03T05:15:32,812 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e6cf43e38e42b2ac74087cf94f8d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733202930200 2024-12-03T05:15:32,813 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e105cdcd8404e7aa60f327d4e27438b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930865 2024-12-03T05:15:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:32,816 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#148 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:32,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/7cc8907b09654de9be439a9e7fb2fdf4 is 175, key is test_row_0/A:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:32,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120391097e4048f54cd3b4c010f5c2bb0d69_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391097e4048f54cd3b4c010f5c2bb0d69_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f8aa1918bb104fa2bce3ac35979e628d, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:32,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f8aa1918bb104fa2bce3ac35979e628d is 175, key is test_row_0/A:col10/1733202931696/Put/seqid=0 2024-12-03T05:15:32,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202992832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202992841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742004_1180 (size=31549) 2024-12-03T05:15:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202992868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202992884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,932 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#151 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:32,940 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f408d741ae604658b25e292b81a0ca1b is 50, key is test_row_0/C:col10/1733202931502/Put/seqid=0 2024-12-03T05:15:32,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742005_1181 (size=31255) 2024-12-03T05:15:32,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202992968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742006_1182 (size=12561) 2024-12-03T05:15:33,006 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f408d741ae604658b25e292b81a0ca1b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f408d741ae604658b25e292b81a0ca1b 2024-12-03T05:15:33,033 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into f408d741ae604658b25e292b81a0ca1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:33,033 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:33,033 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=13, startTime=1733202932600; duration=0sec 2024-12-03T05:15:33,036 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:33,036 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202993196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:33,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/7cc8907b09654de9be439a9e7fb2fdf4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4 2024-12-03T05:15:33,332 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 7cc8907b09654de9be439a9e7fb2fdf4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:33,332 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:33,332 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202932600; duration=0sec 2024-12-03T05:15:33,333 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:33,333 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:33,364 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f8aa1918bb104fa2bce3ac35979e628d 2024-12-03T05:15:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/cc62b9985d8c4adea401a4277ce27197 is 50, key is test_row_0/B:col10/1733202931696/Put/seqid=0 2024-12-03T05:15:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742007_1183 (size=12301) 2024-12-03T05:15:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202993512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:33,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/cc62b9985d8c4adea401a4277ce27197 2024-12-03T05:15:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/3373d5bc80e245e1aef3b98eb0f2e447 is 50, key is test_row_0/C:col10/1733202931696/Put/seqid=0 2024-12-03T05:15:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202993861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:33,864 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742008_1184 (size=12301) 2024-12-03T05:15:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202993892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:33,896 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:33,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202993930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:33,936 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202994020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:34,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/3373d5bc80e245e1aef3b98eb0f2e447 2024-12-03T05:15:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f8aa1918bb104fa2bce3ac35979e628d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d 2024-12-03T05:15:34,301 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d, entries=150, sequenceid=286, filesize=30.5 K 2024-12-03T05:15:34,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/cc62b9985d8c4adea401a4277ce27197 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197 2024-12-03T05:15:34,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197, entries=150, sequenceid=286, filesize=12.0 K 2024-12-03T05:15:34,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/3373d5bc80e245e1aef3b98eb0f2e447 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447 2024-12-03T05:15:34,334 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447, entries=150, sequenceid=286, filesize=12.0 K 2024-12-03T05:15:34,335 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for b7e2b9d2e05056c58426b072bf89a451 in 1686ms, sequenceid=286, compaction requested=false 2024-12-03T05:15:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T05:15:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T05:15:34,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T05:15:34,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7850 sec 2024-12-03T05:15:34,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.7990 sec 2024-12-03T05:15:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T05:15:34,731 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:34,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T05:15:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:15:34,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:34,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:34,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:15:34,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T05:15:34,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:34,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203678d89448dc74d9aa88f28643b7deefd_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202932820/Put/seqid=0 2024-12-03T05:15:34,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742009_1185 (size=12454) 2024-12-03T05:15:34,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:34,964 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203678d89448dc74d9aa88f28643b7deefd_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203678d89448dc74d9aa88f28643b7deefd_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/83adafbbbb164242907a98e3675c464c, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/83adafbbbb164242907a98e3675c464c is 175, key is test_row_0/A:col10/1733202932820/Put/seqid=0 2024-12-03T05:15:35,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742010_1186 (size=31255) 2024-12-03T05:15:35,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:35,041 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/83adafbbbb164242907a98e3675c464c 2024-12-03T05:15:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:15:35,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/029aca683543424fb169f7e9af8a7fd0 is 50, key is test_row_0/B:col10/1733202932820/Put/seqid=0 2024-12-03T05:15:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742011_1187 (size=12301) 2024-12-03T05:15:35,184 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/029aca683543424fb169f7e9af8a7fd0 2024-12-03T05:15:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/7b49eca4b1624b369750aaec15bf9d38 is 50, key is test_row_0/C:col10/1733202932820/Put/seqid=0 2024-12-03T05:15:35,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742012_1188 (size=12301) 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202935186, max timestamp=1733202935263Num uniques 859; 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202935186, max timestamp=1733202935263Num uniques 859; 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202935186, max timestamp=1733202935263Num uniques 859; 2024-12-03T05:15:35,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:15:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:15:35,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202995387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202995504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,647 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/7b49eca4b1624b369750aaec15bf9d38 2024-12-03T05:15:35,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/83adafbbbb164242907a98e3675c464c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c 2024-12-03T05:15:35,709 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c, entries=150, sequenceid=299, filesize=30.5 K 2024-12-03T05:15:35,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202995710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/029aca683543424fb169f7e9af8a7fd0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0 2024-12-03T05:15:35,720 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T05:15:35,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/7b49eca4b1624b369750aaec15bf9d38 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38 2024-12-03T05:15:35,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T05:15:35,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b7e2b9d2e05056c58426b072bf89a451 in 843ms, sequenceid=299, compaction requested=true 2024-12-03T05:15:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T05:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T05:15:35,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T05:15:35,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-12-03T05:15:35,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0050 sec 2024-12-03T05:15:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T05:15:35,880 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:35,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:35,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:35,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T05:15:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:35,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:35,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:35,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:35,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203beb104aa04654f959483a739173283d4_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:35,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202995931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742013_1189 (size=14994) 2024-12-03T05:15:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202995935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202995951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:35,953 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:35,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203beb104aa04654f959483a739173283d4_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beb104aa04654f959483a739173283d4_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:35,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4635d087453a4aaab33bf4f5f3b4a700, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:35,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4635d087453a4aaab33bf4f5f3b4a700 is 175, key is test_row_0/A:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:35,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742014_1190 (size=39949) 2024-12-03T05:15:35,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4635d087453a4aaab33bf4f5f3b4a700 2024-12-03T05:15:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:36,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202996020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/7bf8e0904b9a41c485e10e08d677c63e is 50, key is test_row_0/B:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:36,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:15:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202996051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202996051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742015_1191 (size=12301) 2024-12-03T05:15:36,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/7bf8e0904b9a41c485e10e08d677c63e 2024-12-03T05:15:36,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/af1f3de6b00a4f37a8753fc5b560ff08 is 50, key is test_row_0/C:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:36,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742016_1192 (size=12301) 2024-12-03T05:15:36,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/af1f3de6b00a4f37a8753fc5b560ff08 2024-12-03T05:15:36,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/4635d087453a4aaab33bf4f5f3b4a700 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700 2024-12-03T05:15:36,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700, entries=200, sequenceid=336, filesize=39.0 K 2024-12-03T05:15:36,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/7bf8e0904b9a41c485e10e08d677c63e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e 2024-12-03T05:15:36,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T05:15:36,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/af1f3de6b00a4f37a8753fc5b560ff08 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08 2024-12-03T05:15:36,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T05:15:36,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b7e2b9d2e05056c58426b072bf89a451 in 267ms, sequenceid=336, compaction requested=true 2024-12-03T05:15:36,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:36,149 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:36,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:36,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:36,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:36,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:36,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:36,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:36,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:36,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:36,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:36,151 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,151 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d4d1e435a80d423d9bc20519702fbb88, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=48.3 K 2024-12-03T05:15:36,151 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:36,151 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:36,151 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,151 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=130.9 K 2024-12-03T05:15:36,152 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,152 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700] 2024-12-03T05:15:36,154 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d4d1e435a80d423d9bc20519702fbb88, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930865 2024-12-03T05:15:36,155 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc8907b09654de9be439a9e7fb2fdf4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930865 2024-12-03T05:15:36,155 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8aa1918bb104fa2bce3ac35979e628d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733202931683 2024-12-03T05:15:36,155 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting cc62b9985d8c4adea401a4277ce27197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733202931683 2024-12-03T05:15:36,156 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83adafbbbb164242907a98e3675c464c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202932772 2024-12-03T05:15:36,156 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 029aca683543424fb169f7e9af8a7fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202932772 2024-12-03T05:15:36,157 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4635d087453a4aaab33bf4f5f3b4a700, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935276 2024-12-03T05:15:36,157 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf8e0904b9a41c485e10e08d677c63e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935378 2024-12-03T05:15:36,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T05:15:36,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:36,252 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:36,254 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:36,254 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a0b3f1d0d6bc4240a2332bc248e0a934_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:36,254 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/08839076ae3c438e9f2545e807c7c19f is 50, key is test_row_0/B:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:36,256 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a0b3f1d0d6bc4240a2332bc248e0a934_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:36,256 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a0b3f1d0d6bc4240a2332bc248e0a934_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:36,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203568c5f15817246ac8b1eb177306c4daf_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202935910/Put/seqid=0 2024-12-03T05:15:36,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742017_1193 (size=12881) 2024-12-03T05:15:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742018_1194 (size=4469) 2024-12-03T05:15:36,283 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/08839076ae3c438e9f2545e807c7c19f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08839076ae3c438e9f2545e807c7c19f 2024-12-03T05:15:36,284 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#161 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:36,285 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f6431376e12f444082529e7b41ae3795 is 175, key is test_row_0/A:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:36,291 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 08839076ae3c438e9f2545e807c7c19f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:36,291 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:36,291 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=12, startTime=1733202936149; duration=0sec 2024-12-03T05:15:36,292 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:36,292 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:36,292 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:15:36,295 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:15:36,295 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:36,295 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:36,296 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f408d741ae604658b25e292b81a0ca1b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=48.3 K 2024-12-03T05:15:36,296 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f408d741ae604658b25e292b81a0ca1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733202930865 2024-12-03T05:15:36,297 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3373d5bc80e245e1aef3b98eb0f2e447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733202931683 2024-12-03T05:15:36,297 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b49eca4b1624b369750aaec15bf9d38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733202932772 2024-12-03T05:15:36,298 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting af1f3de6b00a4f37a8753fc5b560ff08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935378 2024-12-03T05:15:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742019_1195 (size=12454) 2024-12-03T05:15:36,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742020_1196 (size=31835) 2024-12-03T05:15:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:36,325 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:36,325 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/e48845cb2fdb430fb6e2350b05ba8dfa is 50, key is test_row_0/C:col10/1733202935378/Put/seqid=0 2024-12-03T05:15:36,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733202936308, max timestamp=1733202936328Num uniques 859; 2024-12-03T05:15:36,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:15:36,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202936317, max timestamp=1733202936328Num uniques 859; 2024-12-03T05:15:36,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202936317, max timestamp=1733202936333Num uniques 859; 2024-12-03T05:15:36,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:36,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:36,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742021_1197 (size=12847) 2024-12-03T05:15:36,337 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203568c5f15817246ac8b1eb177306c4daf_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203568c5f15817246ac8b1eb177306c4daf_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/19b47e42098d4b0a8e1d7da3f8490a74, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/19b47e42098d4b0a8e1d7da3f8490a74 is 175, key is test_row_0/A:col10/1733202935910/Put/seqid=0 2024-12-03T05:15:36,356 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f6431376e12f444082529e7b41ae3795 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795 2024-12-03T05:15:36,363 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into f6431376e12f444082529e7b41ae3795(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:36,363 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:36,363 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=12, startTime=1733202936149; duration=0sec 2024-12-03T05:15:36,364 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:36,364 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:36,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742022_1198 (size=31255) 2024-12-03T05:15:36,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202996425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202996426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:36,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50662 deadline: 1733202996531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202996532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202996532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,535 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202996533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202996741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202996741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:36,747 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/e48845cb2fdb430fb6e2350b05ba8dfa as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e48845cb2fdb430fb6e2350b05ba8dfa 2024-12-03T05:15:36,754 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into e48845cb2fdb430fb6e2350b05ba8dfa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:36,754 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:36,754 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=12, startTime=1733202936150; duration=0sec 2024-12-03T05:15:36,754 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:36,754 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:36,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/19b47e42098d4b0a8e1d7da3f8490a74 2024-12-03T05:15:36,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/53f7721c71cb4bb2adc9978033fb5455 is 50, key is test_row_0/B:col10/1733202935910/Put/seqid=0 2024-12-03T05:15:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742023_1199 (size=12301) 2024-12-03T05:15:36,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/53f7721c71cb4bb2adc9978033fb5455 2024-12-03T05:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ce48b3a192d444d6931cb5867664e63b is 50, key is test_row_0/C:col10/1733202935910/Put/seqid=0 2024-12-03T05:15:36,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742024_1200 (size=12301) 2024-12-03T05:15:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:37,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202997052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202997052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ce48b3a192d444d6931cb5867664e63b 2024-12-03T05:15:37,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/19b47e42098d4b0a8e1d7da3f8490a74 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74 2024-12-03T05:15:37,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74, entries=150, sequenceid=345, filesize=30.5 K 2024-12-03T05:15:37,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/53f7721c71cb4bb2adc9978033fb5455 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455 2024-12-03T05:15:37,336 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T05:15:37,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/ce48b3a192d444d6931cb5867664e63b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b 2024-12-03T05:15:37,344 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T05:15:37,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for b7e2b9d2e05056c58426b072bf89a451 in 1148ms, sequenceid=345, compaction requested=false 2024-12-03T05:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T05:15:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T05:15:37,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T05:15:37,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-12-03T05:15:37,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4750 sec 2024-12-03T05:15:37,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733202936382, max timestamp=1733202937553Num uniques 1118; 2024-12-03T05:15:37,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T05:15:37,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733202936382, max timestamp=1733202937553Num uniques 1018; 2024-12-03T05:15:37,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:15:37,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:15:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:37,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a54489b0ae2646b78b852dbefe2afe80_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202997609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202997610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202997616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742025_1201 (size=14994) 2024-12-03T05:15:37,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:37,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a54489b0ae2646b78b852dbefe2afe80_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a54489b0ae2646b78b852dbefe2afe80_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:37,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f667aa06b834409f92d75486507ed583, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f667aa06b834409f92d75486507ed583 is 175, key is test_row_0/A:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742026_1202 (size=39949) 2024-12-03T05:15:37,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f667aa06b834409f92d75486507ed583 2024-12-03T05:15:37,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/15e02c0025ba48709f5c5f305967f38c is 50, key is test_row_0/B:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742027_1203 (size=12301) 2024-12-03T05:15:37,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/15e02c0025ba48709f5c5f305967f38c 2024-12-03T05:15:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202997721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202997721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202997730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:37,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/508804504c254f7491d2d53499d4c298 is 50, key is test_row_0/C:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742028_1204 (size=12301) 2024-12-03T05:15:37,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/508804504c254f7491d2d53499d4c298 2024-12-03T05:15:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f667aa06b834409f92d75486507ed583 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583 2024-12-03T05:15:37,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583, entries=200, sequenceid=386, filesize=39.0 K 2024-12-03T05:15:37,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/15e02c0025ba48709f5c5f305967f38c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c 2024-12-03T05:15:37,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c, entries=150, sequenceid=386, filesize=12.0 K 2024-12-03T05:15:37,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/508804504c254f7491d2d53499d4c298 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298 2024-12-03T05:15:37,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298, entries=150, sequenceid=386, filesize=12.0 K 2024-12-03T05:15:37,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for b7e2b9d2e05056c58426b072bf89a451 in 222ms, sequenceid=386, compaction requested=true 2024-12-03T05:15:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:37,783 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:15:37,783 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:37,786 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:37,786 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:37,786 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:37,786 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08839076ae3c438e9f2545e807c7c19f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.6 K 2024-12-03T05:15:37,786 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:37,786 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:37,786 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:37,786 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=100.6 K 2024-12-03T05:15:37,786 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:37,787 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583] 2024-12-03T05:15:37,787 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 08839076ae3c438e9f2545e807c7c19f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935378 2024-12-03T05:15:37,787 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6431376e12f444082529e7b41ae3795, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935378 2024-12-03T05:15:37,787 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f7721c71cb4bb2adc9978033fb5455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733202935898 2024-12-03T05:15:37,787 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b47e42098d4b0a8e1d7da3f8490a74, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733202935898 2024-12-03T05:15:37,788 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 15e02c0025ba48709f5c5f305967f38c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:37,788 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f667aa06b834409f92d75486507ed583, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:37,810 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:37,811 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/ba97b349b0724c82a37490280f59bcef is 50, key is test_row_0/B:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,813 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412032a9cbc6ba8684e9aa1ef111b5e252d28_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,818 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412032a9cbc6ba8684e9aa1ef111b5e252d28_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,819 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032a9cbc6ba8684e9aa1ef111b5e252d28_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742029_1205 (size=12983) 2024-12-03T05:15:37,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742030_1206 (size=4469) 2024-12-03T05:15:37,865 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/ba97b349b0724c82a37490280f59bcef as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/ba97b349b0724c82a37490280f59bcef 2024-12-03T05:15:37,872 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into ba97b349b0724c82a37490280f59bcef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:37,872 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:37,872 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202937783; duration=0sec 2024-12-03T05:15:37,873 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:37,873 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:37,873 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:37,874 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:37,874 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:37,874 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:37,874 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e48845cb2fdb430fb6e2350b05ba8dfa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.6 K 2024-12-03T05:15:37,875 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e48845cb2fdb430fb6e2350b05ba8dfa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733202935378 2024-12-03T05:15:37,875 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ce48b3a192d444d6931cb5867664e63b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733202935898 2024-12-03T05:15:37,875 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 508804504c254f7491d2d53499d4c298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:37,898 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#171 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:37,898 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/e5f849b748dd4995aa43b68a3acff47c is 50, key is test_row_0/C:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:37,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742031_1207 (size=12949) 2024-12-03T05:15:37,912 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/e5f849b748dd4995aa43b68a3acff47c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e5f849b748dd4995aa43b68a3acff47c 2024-12-03T05:15:37,920 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into e5f849b748dd4995aa43b68a3acff47c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:37,920 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:37,920 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=13, startTime=1733202937783; duration=0sec 2024-12-03T05:15:37,920 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:37,920 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:37,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202937606, max timestamp=1733202937932Num uniques 859; 2024-12-03T05:15:37,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:37,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202937606, max timestamp=1733202937932Num uniques 859; 2024-12-03T05:15:37,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:37,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-12-03T05:15:37,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:37,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202937606, max timestamp=1733202937934Num uniques 859; 2024-12-03T05:15:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:37,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:37,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030dd2777364314d9e9ac962bd9c8e42ad_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202937611/Put/seqid=0 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202937956, max timestamp=1733202937974Num uniques 859; 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202937956, max timestamp=1733202937974Num uniques 859; 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202937956, max timestamp=1733202937974Num uniques 859; 2024-12-03T05:15:37,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:37,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742032_1208 (size=17534) 2024-12-03T05:15:37,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:37,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030dd2777364314d9e9ac962bd9c8e42ad_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030dd2777364314d9e9ac962bd9c8e42ad_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:37,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b4dae4176bb7413fbfcf4f6c124437f4, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:37,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b4dae4176bb7413fbfcf4f6c124437f4 is 175, key is test_row_0/A:col10/1733202937611/Put/seqid=0 2024-12-03T05:15:38,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742033_1209 (size=48639) 2024-12-03T05:15:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202998007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202998021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202998008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b4dae4176bb7413fbfcf4f6c124437f4 2024-12-03T05:15:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T05:15:38,041 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:38,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T05:15:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:38,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:38,045 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:38,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:38,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/5952c22328294d989fe8b9b66ff5ba9d is 50, key is test_row_0/B:col10/1733202937611/Put/seqid=0 2024-12-03T05:15:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742034_1210 (size=12301) 2024-12-03T05:15:38,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/5952c22328294d989fe8b9b66ff5ba9d 2024-12-03T05:15:38,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/1707455131f84c298a268aa24876a947 is 50, key is test_row_0/C:col10/1733202937611/Put/seqid=0 2024-12-03T05:15:38,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742035_1211 (size=12301) 2024-12-03T05:15:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202998131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202998131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202998131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:38,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,254 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#170 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:38,254 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/776fa6a9f44f494392c2d83f67a5c56e is 175, key is test_row_0/A:col10/1733202936418/Put/seqid=0 2024-12-03T05:15:38,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742036_1212 (size=31937) 2024-12-03T05:15:38,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202998340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202998341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202998341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:38,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/1707455131f84c298a268aa24876a947 2024-12-03T05:15:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b4dae4176bb7413fbfcf4f6c124437f4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4 2024-12-03T05:15:38,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4, entries=250, sequenceid=410, filesize=47.5 K 2024-12-03T05:15:38,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/5952c22328294d989fe8b9b66ff5ba9d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d 2024-12-03T05:15:38,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T05:15:38,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/1707455131f84c298a268aa24876a947 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947 2024-12-03T05:15:38,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T05:15:38,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for b7e2b9d2e05056c58426b072bf89a451 in 626ms, sequenceid=410, compaction requested=false 2024-12-03T05:15:38,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:38,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-12-03T05:15:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:15:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:15:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:38,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:15:38,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202998664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:38,673 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/776fa6a9f44f494392c2d83f67a5c56e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e 2024-12-03T05:15:38,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120397af45f239af4443a18ae303e6633dd2_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202998667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202998669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,680 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 776fa6a9f44f494392c2d83f67a5c56e(size=31.2 K), total size for store is 78.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:38,680 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:38,680 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202937782; duration=0sec 2024-12-03T05:15:38,680 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:38,680 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:38,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742037_1213 (size=14994) 2024-12-03T05:15:38,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120397af45f239af4443a18ae303e6633dd2_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120397af45f239af4443a18ae303e6633dd2_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:38,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/e42fde2efab3410298f66fb7aa2e603e, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:38,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/e42fde2efab3410298f66fb7aa2e603e is 175, key is test_row_0/A:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742038_1214 (size=39949) 2024-12-03T05:15:38,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/e42fde2efab3410298f66fb7aa2e603e 2024-12-03T05:15:38,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/86c9475da24f485a84dcbf56e4f23f8c is 50, key is test_row_0/B:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742039_1215 (size=12301) 2024-12-03T05:15:38,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/86c9475da24f485a84dcbf56e4f23f8c 2024-12-03T05:15:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202998781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202998781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202998784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:38,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/9c7a24d3a79c4a529d404b86c2b21439 is 50, key is test_row_0/C:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:38,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742040_1216 (size=12301) 2024-12-03T05:15:38,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/9c7a24d3a79c4a529d404b86c2b21439 2024-12-03T05:15:38,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/e42fde2efab3410298f66fb7aa2e603e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e 2024-12-03T05:15:38,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e, entries=200, sequenceid=448, filesize=39.0 K 2024-12-03T05:15:38,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/86c9475da24f485a84dcbf56e4f23f8c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c 2024-12-03T05:15:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c, entries=150, sequenceid=448, filesize=12.0 K 2024-12-03T05:15:38,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/9c7a24d3a79c4a529d404b86c2b21439 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439 2024-12-03T05:15:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439, entries=150, sequenceid=448, filesize=12.0 K 2024-12-03T05:15:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for b7e2b9d2e05056c58426b072bf89a451 in 233ms, sequenceid=448, compaction requested=true 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:38,887 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:38,887 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:38,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:38,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,889 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:38,889 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120525 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:38,889 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:38,889 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:38,889 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,889 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,890 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=117.7 K 2024-12-03T05:15:38,890 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/ba97b349b0724c82a37490280f59bcef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.7 K 2024-12-03T05:15:38,890 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,890 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e] 2024-12-03T05:15:38,890 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba97b349b0724c82a37490280f59bcef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:38,891 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 776fa6a9f44f494392c2d83f67a5c56e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,891 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5952c22328294d989fe8b9b66ff5ba9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733202937608 2024-12-03T05:15:38,892 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b4dae4176bb7413fbfcf4f6c124437f4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733202937606 2024-12-03T05:15:38,892 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c9475da24f485a84dcbf56e4f23f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,893 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e42fde2efab3410298f66fb7aa2e603e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,917 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:38,917 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:38,917 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b is 50, key is test_row_0/B:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,937 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203dc9ec393b1a741d39a44a088e25dfeab_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,939 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203dc9ec393b1a741d39a44a088e25dfeab_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:38,939 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dc9ec393b1a741d39a44a088e25dfeab_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742041_1217 (size=13085) 2024-12-03T05:15:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,967 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b 2024-12-03T05:15:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T05:15:38,978 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 0f7d806fb87b4c1fb05dc85c3d1a1c1b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:38,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:38,978 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202938887; duration=0sec 2024-12-03T05:15:38,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:38,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,979 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T05:15:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:38,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:38,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:38,980 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:38,980 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:38,980 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:38,981 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e5f849b748dd4995aa43b68a3acff47c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.7 K 2024-12-03T05:15:38,982 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f849b748dd4995aa43b68a3acff47c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733202936382 2024-12-03T05:15:38,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742042_1218 (size=4469) 2024-12-03T05:15:38,984 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1707455131f84c298a268aa24876a947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733202937608 2024-12-03T05:15:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,984 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7a24d3a79c4a529d404b86c2b21439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,988 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#179 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:38,989 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d4a18e2d7dab438cb541be10750f905f is 175, key is test_row_0/A:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:38,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cad3927c52ea46be93d29cf9fab9694e_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202938665/Put/seqid=0 2024-12-03T05:15:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:39,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,024 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:39,024 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/458387ae29254123a18c5400ec155a38 is 50, key is test_row_0/C:col10/1733202938653/Put/seqid=0 2024-12-03T05:15:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742044_1220 (size=9914) 2024-12-03T05:15:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,044 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cad3927c52ea46be93d29cf9fab9694e_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cad3927c52ea46be93d29cf9fab9694e_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/977d064e372742078d999c6e4868a8e1, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/977d064e372742078d999c6e4868a8e1 is 175, key is test_row_0/A:col10/1733202938665/Put/seqid=0 2024-12-03T05:15:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742043_1219 (size=32039) 2024-12-03T05:15:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202939044, max timestamp=1733202939071Num uniques 859; 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202939044, max timestamp=1733202939071Num uniques 859; 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202939044, max timestamp=1733202939071Num uniques 859; 2024-12-03T05:15:39,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:15:39,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742045_1221 (size=13051) 2024-12-03T05:15:39,089 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/458387ae29254123a18c5400ec155a38 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/458387ae29254123a18c5400ec155a38 2024-12-03T05:15:39,097 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into 458387ae29254123a18c5400ec155a38(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:39,097 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:39,097 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=13, startTime=1733202938887; duration=0sec 2024-12-03T05:15:39,097 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:39,097 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:39,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742046_1222 (size=22561) 2024-12-03T05:15:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202999134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733202939096, max timestamp=1733202939137Num uniques 1168; 2024-12-03T05:15:39,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T05:15:39,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733202939096, max timestamp=1733202939137Num uniques 1168; 2024-12-03T05:15:39,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733202939096, max timestamp=1733202939137Num uniques 1168; 2024-12-03T05:15:39,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T05:15:39,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T05:15:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:39,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733202939203, max timestamp=1733202939220Num uniques 1077; 2024-12-03T05:15:39,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733202939203, max timestamp=1733202939220Num uniques 1077; 2024-12-03T05:15:39,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-03T05:15:39,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-03T05:15:39,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733202939203, max timestamp=1733202939220Num uniques 1077; 2024-12-03T05:15:39,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-03T05:15:39,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202939265, max timestamp=1733202939272Num uniques 1036; 2024-12-03T05:15:39,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:15:39,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202939265, max timestamp=1733202939272Num uniques 1036; 2024-12-03T05:15:39,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:15:39,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202939265, max timestamp=1733202939272Num uniques 1036; 2024-12-03T05:15:39,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733202939313, max timestamp=1733202939339Num uniques 1045; 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733202939313, max timestamp=1733202939339Num uniques 1045; 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733202939313, max timestamp=1733202939339Num uniques 1045; 2024-12-03T05:15:39,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202939399, max timestamp=1733202939419Num uniques 1104; 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202939399, max timestamp=1733202939419Num uniques 1104; 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202939399, max timestamp=1733202939419Num uniques 1104; 2024-12-03T05:15:39,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T05:15:39,463 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/d4a18e2d7dab438cb541be10750f905f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f 2024-12-03T05:15:39,476 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into d4a18e2d7dab438cb541be10750f905f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:39,476 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:39,476 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202938887; duration=0sec 2024-12-03T05:15:39,477 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:39,477 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:39,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733202939463, max timestamp=1733202939483Num uniques 1013; 2024-12-03T05:15:39,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733202939463, max timestamp=1733202939483Num uniques 1013; 2024-12-03T05:15:39,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T05:15:39,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T05:15:39,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733202939463, max timestamp=1733202939483Num uniques 1013; 2024-12-03T05:15:39,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T05:15:39,500 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/977d064e372742078d999c6e4868a8e1 2024-12-03T05:15:39,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/43f12732787549168bafcdb3e7fed6d5 is 50, key is test_row_0/B:col10/1733202938665/Put/seqid=0 2024-12-03T05:15:39,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733202939526, max timestamp=1733202939542Num uniques 1072; 2024-12-03T05:15:39,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-03T05:15:39,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733202939526, max timestamp=1733202939542Num uniques 1072; 2024-12-03T05:15:39,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-03T05:15:39,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202939526, max timestamp=1733202939542Num uniques 1072; 2024-12-03T05:15:39,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=6870, suffix heap size=139456,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=150 2024-12-03T05:15:39,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742047_1223 (size=9857) 2024-12-03T05:15:39,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/43f12732787549168bafcdb3e7fed6d5 2024-12-03T05:15:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/8fc7ae36f2e4478bb2ff1f74f72066a8 is 50, key is test_row_0/C:col10/1733202938665/Put/seqid=0 2024-12-03T05:15:39,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742048_1224 (size=9857) 2024-12-03T05:15:39,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/8fc7ae36f2e4478bb2ff1f74f72066a8 2024-12-03T05:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/977d064e372742078d999c6e4868a8e1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1 2024-12-03T05:15:39,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1, entries=100, sequenceid=457, filesize=22.0 K 2024-12-03T05:15:39,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/43f12732787549168bafcdb3e7fed6d5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5 2024-12-03T05:15:39,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5, entries=100, sequenceid=457, filesize=9.6 K 2024-12-03T05:15:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/8fc7ae36f2e4478bb2ff1f74f72066a8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8 2024-12-03T05:15:39,621 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8, entries=100, sequenceid=457, filesize=9.6 K 2024-12-03T05:15:39,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733202939593, max timestamp=1733202939608Num uniques 1031; 2024-12-03T05:15:39,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T05:15:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733202939593, max timestamp=1733202939608Num uniques 1009; 2024-12-03T05:15:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733202939593, max timestamp=1733202939608Num uniques 1031; 2024-12-03T05:15:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8286, suffix heap size=132904,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=181 2024-12-03T05:15:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-03T05:15:39,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=146.62 KB/150138 for b7e2b9d2e05056c58426b072bf89a451 in 643ms, sequenceid=457, compaction requested=false 2024-12-03T05:15:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:39,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T05:15:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T05:15:39,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T05:15:39,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-12-03T05:15:39,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5870 sec 2024-12-03T05:15:39,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=49.20 KB heapSize=129.02 KB 2024-12-03T05:15:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:39,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:39,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:39,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:39,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:39,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:39,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff46147a588c4996b80c556f424628fe_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742049_1225 (size=14994) 2024-12-03T05:15:39,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:39,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202939696, max timestamp=1733202939703Num uniques 859; 2024-12-03T05:15:39,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:39,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202939696, max timestamp=1733202939703Num uniques 859; 2024-12-03T05:15:39,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:39,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202939696, max timestamp=1733202939703Num uniques 859; 2024-12-03T05:15:39,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:39,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff46147a588c4996b80c556f424628fe_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff46147a588c4996b80c556f424628fe_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:39,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/3b8c7074f51743b381da65180cbdeffb, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:39,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/3b8c7074f51743b381da65180cbdeffb is 175, key is test_row_0/A:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:39,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742050_1226 (size=39949) 2024-12-03T05:15:39,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=621, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/3b8c7074f51743b381da65180cbdeffb 2024-12-03T05:15:39,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/6e0196a26e8b4bbb9ff27dba57bf2c22 is 50, key is test_row_0/B:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:39,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742051_1227 (size=12301) 2024-12-03T05:15:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202999750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202999754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202999756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733202999860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733202999871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733202999871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:39,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50654 deadline: 1733202999962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:39,970 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in b7e2b9d2e05056c58426b072bf89a451 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203000071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203000080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203000080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/6e0196a26e8b4bbb9ff27dba57bf2c22 2024-12-03T05:15:40,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/29566c6ff2b8421da59bc71ccc876139 is 50, key is test_row_0/C:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T05:15:40,192 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:40,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-03T05:15:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:15:40,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:40,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:40,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:40,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742052_1228 (size=12301) 2024-12-03T05:15:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:15:40,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:15:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203000381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203000390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203000390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:15:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:15:40,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/29566c6ff2b8421da59bc71ccc876139 2024-12-03T05:15:40,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/3b8c7074f51743b381da65180cbdeffb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb 2024-12-03T05:15:40,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:15:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:40,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb, entries=200, sequenceid=621, filesize=39.0 K 2024-12-03T05:15:40,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/6e0196a26e8b4bbb9ff27dba57bf2c22 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22 2024-12-03T05:15:40,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22, entries=150, sequenceid=621, filesize=12.0 K 2024-12-03T05:15:40,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/29566c6ff2b8421da59bc71ccc876139 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139 2024-12-03T05:15:40,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139, entries=150, sequenceid=621, filesize=12.0 K 2024-12-03T05:15:40,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~55.91 KB/57250, heapSize ~148.89 KB/152464, currentSize=127.47 KB/130530 for b7e2b9d2e05056c58426b072bf89a451 in 1060ms, sequenceid=621, compaction requested=true 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:40,695 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:40,695 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:40,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:40,701 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:40,701 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,701 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=34.4 K 2024-12-03T05:15:40,702 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:40,702 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:40,702 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,702 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=92.3 K 2024-12-03T05:15:40,702 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,702 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb] 2024-12-03T05:15:40,703 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7d806fb87b4c1fb05dc85c3d1a1c1b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:40,704 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a18e2d7dab438cb541be10750f905f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:40,705 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 977d064e372742078d999c6e4868a8e1, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733202938663 2024-12-03T05:15:40,705 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f12732787549168bafcdb3e7fed6d5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733202938663 2024-12-03T05:15:40,705 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8c7074f51743b381da65180cbdeffb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939608 2024-12-03T05:15:40,705 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0196a26e8b4bbb9ff27dba57bf2c22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939631 2024-12-03T05:15:40,735 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:40,735 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b is 50, key is test_row_0/B:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:40,743 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:40,765 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120310fb13b62fd442e3b2a644d280b20477_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:40,767 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120310fb13b62fd442e3b2a644d280b20477_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:40,767 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120310fb13b62fd442e3b2a644d280b20477_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:40,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742053_1229 (size=13187) 2024-12-03T05:15:40,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742054_1230 (size=4469) 2024-12-03T05:15:40,810 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#188 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:40,811 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/77971d0adc5c4b75b4ecc4fe1b89f0a6 is 175, key is test_row_0/A:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:40,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,821 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:40,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742055_1231 (size=32141) 2024-12-03T05:15:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:15:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dbf0b687156f4522b30a76f66bb71b66_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202939752/Put/seqid=0 2024-12-03T05:15:40,854 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/77971d0adc5c4b75b4ecc4fe1b89f0a6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6 2024-12-03T05:15:40,862 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 77971d0adc5c4b75b4ecc4fe1b89f0a6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:40,862 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:40,862 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202940695; duration=0sec 2024-12-03T05:15:40,862 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:40,862 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:40,862 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:40,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742056_1232 (size=12454) 2024-12-03T05:15:40,863 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:40,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:40,864 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:40,864 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/458387ae29254123a18c5400ec155a38, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=34.4 K 2024-12-03T05:15:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:40,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 458387ae29254123a18c5400ec155a38, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733202938005 2024-12-03T05:15:40,865 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc7ae36f2e4478bb2ff1f74f72066a8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733202938663 2024-12-03T05:15:40,866 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29566c6ff2b8421da59bc71ccc876139, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939631 2024-12-03T05:15:40,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dbf0b687156f4522b30a76f66bb71b66_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbf0b687156f4522b30a76f66bb71b66_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:40,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/be7c1fd3c1d440779f90f4bc1b9f8d19, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:40,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/be7c1fd3c1d440779f90f4bc1b9f8d19 is 175, key is test_row_0/A:col10/1733202939752/Put/seqid=0 2024-12-03T05:15:40,882 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:40,882 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f8bd5c21682c4d43a57536bbd4c33c3a is 50, key is test_row_0/C:col10/1733202939632/Put/seqid=0 2024-12-03T05:15:40,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:40,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742057_1233 (size=31255) 2024-12-03T05:15:40,917 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=657, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/be7c1fd3c1d440779f90f4bc1b9f8d19 2024-12-03T05:15:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203000919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203000925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203000926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/bef14439e1d641c787f1a527322f6784 is 50, key is test_row_0/B:col10/1733202939752/Put/seqid=0 2024-12-03T05:15:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742059_1235 (size=12301) 2024-12-03T05:15:40,945 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/bef14439e1d641c787f1a527322f6784 2024-12-03T05:15:40,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742058_1234 (size=13153) 2024-12-03T05:15:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/c63c8b9d33044c0484f5fa8eb58d14d6 is 50, key is test_row_0/C:col10/1733202939752/Put/seqid=0 2024-12-03T05:15:40,962 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f8bd5c21682c4d43a57536bbd4c33c3a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f8bd5c21682c4d43a57536bbd4c33c3a 2024-12-03T05:15:40,976 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into f8bd5c21682c4d43a57536bbd4c33c3a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:40,976 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:40,976 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=13, startTime=1733202940695; duration=0sec 2024-12-03T05:15:40,977 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:40,977 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:40,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742060_1236 (size=12301) 2024-12-03T05:15:40,990 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/c63c8b9d33044c0484f5fa8eb58d14d6 2024-12-03T05:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/be7c1fd3c1d440779f90f4bc1b9f8d19 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19 2024-12-03T05:15:41,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19, entries=150, sequenceid=657, filesize=30.5 K 2024-12-03T05:15:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/bef14439e1d641c787f1a527322f6784 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784 2024-12-03T05:15:41,027 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784, entries=150, sequenceid=657, filesize=12.0 K 2024-12-03T05:15:41,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/c63c8b9d33044c0484f5fa8eb58d14d6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6 2024-12-03T05:15:41,035 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6, entries=150, sequenceid=657, filesize=12.0 K 2024-12-03T05:15:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203001031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,037 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b7e2b9d2e05056c58426b072bf89a451 in 216ms, sequenceid=657, compaction requested=false 2024-12-03T05:15:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-03T05:15:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-03T05:15:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:41,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:41,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T05:15:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 837 msec 2024-12-03T05:15:41,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad7de245002045e8a9244e3853817acf_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:41,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 850 msec 2024-12-03T05:15:41,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742061_1237 (size=14994) 2024-12-03T05:15:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203001092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203001096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,180 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b 2024-12-03T05:15:41,189 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 005e0e1bba4a4b2d9f8d11a9d6b91b8b(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:41,189 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:41,189 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202940695; duration=0sec 2024-12-03T05:15:41,189 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:41,189 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203001201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203001211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203001241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T05:15:41,353 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:41,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-03T05:15:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:41,359 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:41,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:41,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:41,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203001412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203001423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:41,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:41,468 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad7de245002045e8a9244e3853817acf_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad7de245002045e8a9244e3853817acf_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:41,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f71be2b364d14d40bae9cd1ad1c79b2e, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:41,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f71be2b364d14d40bae9cd1ad1c79b2e is 175, key is test_row_0/A:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:41,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742062_1238 (size=39949) 2024-12-03T05:15:41,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=671, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f71be2b364d14d40bae9cd1ad1c79b2e 2024-12-03T05:15:41,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/012ea60f2b214daeb469009014d2c42b is 50, key is test_row_0/B:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:41,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:41,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:41,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742063_1239 (size=12301) 2024-12-03T05:15:41,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/012ea60f2b214daeb469009014d2c42b 2024-12-03T05:15:41,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/350e89fff41047e0a9e8392e4c8fdee0 is 50, key is test_row_0/C:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50610 deadline: 1733203001554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742064_1240 (size=12301) 2024-12-03T05:15:41,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50592 deadline: 1733203001720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50612 deadline: 1733203001732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:15:41,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/350e89fff41047e0a9e8392e4c8fdee0 2024-12-03T05:15:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/f71be2b364d14d40bae9cd1ad1c79b2e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e 2024-12-03T05:15:41,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:41,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e, entries=200, sequenceid=671, filesize=39.0 K 2024-12-03T05:15:42,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/012ea60f2b214daeb469009014d2c42b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b 2024-12-03T05:15:42,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b, entries=150, sequenceid=671, filesize=12.0 K 2024-12-03T05:15:42,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/350e89fff41047e0a9e8392e4c8fdee0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0 2024-12-03T05:15:42,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0, entries=150, sequenceid=671, filesize=12.0 K 2024-12-03T05:15:42,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b7e2b9d2e05056c58426b072bf89a451 in 986ms, sequenceid=671, compaction requested=true 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:42,028 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:42,028 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7e2b9d2e05056c58426b072bf89a451:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:42,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:42,030 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:42,030 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/A is initiating minor compaction (all files) 2024-12-03T05:15:42,030 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/A in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,030 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=100.9 K 2024-12-03T05:15:42,030 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,030 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e] 2024-12-03T05:15:42,030 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:42,030 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/B is initiating minor compaction (all files) 2024-12-03T05:15:42,030 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/B in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,031 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.9 K 2024-12-03T05:15:42,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77971d0adc5c4b75b4ecc4fe1b89f0a6, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939631 2024-12-03T05:15:42,032 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 005e0e1bba4a4b2d9f8d11a9d6b91b8b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939631 2024-12-03T05:15:42,032 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7c1fd3c1d440779f90f4bc1b9f8d19, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733202939730 2024-12-03T05:15:42,032 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bef14439e1d641c787f1a527322f6784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733202939730 2024-12-03T05:15:42,032 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71be2b364d14d40bae9cd1ad1c79b2e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202940910 2024-12-03T05:15:42,032 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 012ea60f2b214daeb469009014d2c42b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202940910 2024-12-03T05:15:42,041 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:42,042 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/97f2b43f70de4795980d8cd6e74ccc1c is 50, key is test_row_0/B:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:42,043 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:42,048 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120346488d141d574a3ba8eece3ba13acd6e_b7e2b9d2e05056c58426b072bf89a451 store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:42,053 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120346488d141d574a3ba8eece3ba13acd6e_b7e2b9d2e05056c58426b072bf89a451, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:42,053 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120346488d141d574a3ba8eece3ba13acd6e_b7e2b9d2e05056c58426b072bf89a451 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202941072, max timestamp=1733202942063Num uniques 859; 2024-12-03T05:15:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202941072, max timestamp=1733202942063Num uniques 859; 2024-12-03T05:15:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:42,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202941072, max timestamp=1733202942063Num uniques 859; 2024-12-03T05:15:42,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:42,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742065_1241 (size=13289) 2024-12-03T05:15:42,085 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/97f2b43f70de4795980d8cd6e74ccc1c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/97f2b43f70de4795980d8cd6e74ccc1c 2024-12-03T05:15:42,086 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-12-03T05:15:42,087 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,087 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,087 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,087 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-12-03T05:15:42,087 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,087 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,088 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742066_1242 (size=4469) 2024-12-03T05:15:42,088 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-12-03T05:15:42,088 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,088 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,088 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,089 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-12-03T05:15:42,090 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,090 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,090 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,092 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#A#compaction#197 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:42,092 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/22af270daed24052b4a29f4b7782e4eb is 175, key is test_row_0/A:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:42,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395be59aa74de441c8247924141df866b_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202941072/Put/seqid=0 2024-12-03T05:15:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742067_1243 (size=32243) 2024-12-03T05:15:42,098 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/B of b7e2b9d2e05056c58426b072bf89a451 into 97f2b43f70de4795980d8cd6e74ccc1c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:42,098 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:42,098 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/B, priority=13, startTime=1733202942028; duration=0sec 2024-12-03T05:15:42,098 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:42,098 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:B 2024-12-03T05:15:42,098 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:42,098 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-12-03T05:15:42,098 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,098 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,098 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,100 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:42,100 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): b7e2b9d2e05056c58426b072bf89a451/C is initiating minor compaction (all files) 2024-12-03T05:15:42,100 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7e2b9d2e05056c58426b072bf89a451/C in TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,100 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f8bd5c21682c4d43a57536bbd4c33c3a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp, totalSize=36.9 K 2024-12-03T05:15:42,101 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f8bd5c21682c4d43a57536bbd4c33c3a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1733202939631 2024-12-03T05:15:42,101 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c63c8b9d33044c0484f5fa8eb58d14d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733202939730 2024-12-03T05:15:42,101 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 350e89fff41047e0a9e8392e4c8fdee0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202940910 2024-12-03T05:15:42,103 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/22af270daed24052b4a29f4b7782e4eb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/22af270daed24052b4a29f4b7782e4eb 2024-12-03T05:15:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742068_1244 (size=14994) 2024-12-03T05:15:42,108 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7e2b9d2e05056c58426b072bf89a451#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:42,109 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f9710230e75c42df83c41234287c4767 is 50, key is test_row_0/C:col10/1733202940910/Put/seqid=0 2024-12-03T05:15:42,110 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/A of b7e2b9d2e05056c58426b072bf89a451 into 22af270daed24052b4a29f4b7782e4eb(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:42,110 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:42,110 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/A, priority=13, startTime=1733202942028; duration=0sec 2024-12-03T05:15:42,110 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:42,110 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:A 2024-12-03T05:15:42,117 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:42,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395be59aa74de441c8247924141df866b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395be59aa74de441c8247924141df866b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:42,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b70576d5f9534a16a27b4221bf8091fe, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:42,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b70576d5f9534a16a27b4221bf8091fe is 175, key is test_row_0/A:col10/1733202941072/Put/seqid=0 2024-12-03T05:15:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742069_1245 (size=13255) 2024-12-03T05:15:42,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742070_1246 (size=39949) 2024-12-03T05:15:42,243 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-12-03T05:15:42,243 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,243 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,243 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,252 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-12-03T05:15:42,252 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:42,252 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,252 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:42,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:42,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:42,549 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/f9710230e75c42df83c41234287c4767 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f9710230e75c42df83c41234287c4767 2024-12-03T05:15:42,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=693, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b70576d5f9534a16a27b4221bf8091fe 2024-12-03T05:15:42,556 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7e2b9d2e05056c58426b072bf89a451/C of b7e2b9d2e05056c58426b072bf89a451 into f9710230e75c42df83c41234287c4767(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:42,556 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:42,556 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., storeName=b7e2b9d2e05056c58426b072bf89a451/C, priority=13, startTime=1733202942028; duration=0sec 2024-12-03T05:15:42,556 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:42,556 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7e2b9d2e05056c58426b072bf89a451:C 2024-12-03T05:15:42,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/625503ad753145f5b32465fc647bfd3a is 50, key is test_row_0/B:col10/1733202941072/Put/seqid=0 2024-12-03T05:15:42,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742071_1247 (size=12301) 2024-12-03T05:15:42,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:42,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:42,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/625503ad753145f5b32465fc647bfd3a 2024-12-03T05:15:43,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/61c6307b9c4542d9a2affbfff99eb9b2 is 50, key is test_row_0/C:col10/1733202941072/Put/seqid=0 2024-12-03T05:15:43,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742072_1248 (size=12301) 2024-12-03T05:15:43,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. as already flushing 2024-12-03T05:15:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:43,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/61c6307b9c4542d9a2affbfff99eb9b2 2024-12-03T05:15:43,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/b70576d5f9534a16a27b4221bf8091fe as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b70576d5f9534a16a27b4221bf8091fe 2024-12-03T05:15:43,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b70576d5f9534a16a27b4221bf8091fe, entries=200, sequenceid=693, filesize=39.0 K 2024-12-03T05:15:43,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/625503ad753145f5b32465fc647bfd3a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/625503ad753145f5b32465fc647bfd3a 2024-12-03T05:15:43,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/625503ad753145f5b32465fc647bfd3a, entries=150, sequenceid=693, filesize=12.0 K 2024-12-03T05:15:43,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/61c6307b9c4542d9a2affbfff99eb9b2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/61c6307b9c4542d9a2affbfff99eb9b2 2024-12-03T05:15:43,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/61c6307b9c4542d9a2affbfff99eb9b2, entries=150, sequenceid=693, filesize=12.0 K 2024-12-03T05:15:43,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=40.25 KB/41220 for b7e2b9d2e05056c58426b072bf89a451 in 1415ms, sequenceid=693, compaction requested=false 2024-12-03T05:15:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:43,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:43,529 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:43,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120388765c167ce34ad194334f162bd95490_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202942241/Put/seqid=0 2024-12-03T05:15:43,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742073_1249 (size=9914) 2024-12-03T05:15:43,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:43,953 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120388765c167ce34ad194334f162bd95490_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388765c167ce34ad194334f162bd95490_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/5769532c9f16494e9065f2de3355f71a, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:43,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/5769532c9f16494e9065f2de3355f71a is 175, key is test_row_0/A:col10/1733202942241/Put/seqid=0 2024-12-03T05:15:43,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742074_1250 (size=22561) 2024-12-03T05:15:44,360 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=705, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/5769532c9f16494e9065f2de3355f71a 2024-12-03T05:15:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/c38d8e223f5443b2a5062178a25bad00 is 50, key is test_row_0/B:col10/1733202942241/Put/seqid=0 2024-12-03T05:15:44,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742075_1251 (size=9857) 2024-12-03T05:15:44,775 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/c38d8e223f5443b2a5062178a25bad00 2024-12-03T05:15:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/6c32f6806faf49fd88fb43450f3c0332 is 50, key is test_row_0/C:col10/1733202942241/Put/seqid=0 2024-12-03T05:15:44,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742076_1252 (size=9857) 2024-12-03T05:15:45,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/6c32f6806faf49fd88fb43450f3c0332 2024-12-03T05:15:45,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/5769532c9f16494e9065f2de3355f71a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/5769532c9f16494e9065f2de3355f71a 2024-12-03T05:15:45,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/5769532c9f16494e9065f2de3355f71a, entries=100, sequenceid=705, filesize=22.0 K 2024-12-03T05:15:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/c38d8e223f5443b2a5062178a25bad00 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/c38d8e223f5443b2a5062178a25bad00 2024-12-03T05:15:45,203 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/c38d8e223f5443b2a5062178a25bad00, entries=100, sequenceid=705, filesize=9.6 K 2024-12-03T05:15:45,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/6c32f6806faf49fd88fb43450f3c0332 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/6c32f6806faf49fd88fb43450f3c0332 2024-12-03T05:15:45,208 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/6c32f6806faf49fd88fb43450f3c0332, entries=100, sequenceid=705, filesize=9.6 K 2024-12-03T05:15:45,209 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for b7e2b9d2e05056c58426b072bf89a451 in 1680ms, sequenceid=705, compaction requested=true 2024-12-03T05:15:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for b7e2b9d2e05056c58426b072bf89a451: 2024-12-03T05:15:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:45,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-03T05:15:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-03T05:15:45,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T05:15:45,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8510 sec 2024-12-03T05:15:45,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.8590 sec 2024-12-03T05:15:45,482 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:15:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T05:15:45,521 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:46,593 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-12-03T05:15:46,593 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:46,593 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:46,593 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:46,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:50,062 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-12-03T05:15:50,062 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:15:50,062 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:50,062 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:50,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 182 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 198 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2106 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2087 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2981 rows 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-12-03T05:15:50,062 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2987 rows 2024-12-03T05:15:50,062 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:15:50,063 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:15:50,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:50,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:50,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:15:50,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:15:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:50,067 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202950067"}]},"ts":"1733202950067"} 2024-12-03T05:15:50,069 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:15:50,069 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:15:50,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:15:50,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, UNASSIGN}] 2024-12-03T05:15:50,072 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, UNASSIGN 2024-12-03T05:15:50,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:50,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, UNASSIGN because future has completed 2024-12-03T05:15:50,074 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:15:50,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:50,226 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing b7e2b9d2e05056c58426b072bf89a451, disabling compactions & flushes 2024-12-03T05:15:50,227 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. after waiting 0 ms 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:50,227 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing b7e2b9d2e05056c58426b072bf89a451 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=A 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=B 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7e2b9d2e05056c58426b072bf89a451, store=C 2024-12-03T05:15:50,227 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:50,233 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203978f59a0e84540f59c5acd2b1057188d_b7e2b9d2e05056c58426b072bf89a451 is 50, key is test_row_0/A:col10/1733202950061/Put/seqid=0 2024-12-03T05:15:50,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742077_1253 (size=9914) 2024-12-03T05:15:50,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:50,638 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:50,642 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203978f59a0e84540f59c5acd2b1057188d_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203978f59a0e84540f59c5acd2b1057188d_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:50,643 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/0700331340544c58803eb13d6168a21d, store: [table=TestAcidGuarantees family=A region=b7e2b9d2e05056c58426b072bf89a451] 2024-12-03T05:15:50,644 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/0700331340544c58803eb13d6168a21d is 175, key is test_row_0/A:col10/1733202950061/Put/seqid=0 2024-12-03T05:15:50,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742078_1254 (size=22561) 2024-12-03T05:15:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:51,055 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=710, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/0700331340544c58803eb13d6168a21d 2024-12-03T05:15:51,070 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/08467f0ab6e647f6a1db6f270b9c307a is 50, key is test_row_0/B:col10/1733202950061/Put/seqid=0 2024-12-03T05:15:51,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742079_1255 (size=9857) 2024-12-03T05:15:51,100 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/08467f0ab6e647f6a1db6f270b9c307a 2024-12-03T05:15:51,122 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d7c8a4ae9a5643a8a4d34d5ba494e72c is 50, key is test_row_0/C:col10/1733202950061/Put/seqid=0 2024-12-03T05:15:51,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742080_1256 (size=9857) 2024-12-03T05:15:51,135 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d7c8a4ae9a5643a8a4d34d5ba494e72c 2024-12-03T05:15:51,141 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/A/0700331340544c58803eb13d6168a21d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/0700331340544c58803eb13d6168a21d 2024-12-03T05:15:51,150 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/0700331340544c58803eb13d6168a21d, entries=100, sequenceid=710, filesize=22.0 K 2024-12-03T05:15:51,151 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/B/08467f0ab6e647f6a1db6f270b9c307a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08467f0ab6e647f6a1db6f270b9c307a 2024-12-03T05:15:51,158 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08467f0ab6e647f6a1db6f270b9c307a, entries=100, sequenceid=710, filesize=9.6 K 2024-12-03T05:15:51,159 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/.tmp/C/d7c8a4ae9a5643a8a4d34d5ba494e72c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d7c8a4ae9a5643a8a4d34d5ba494e72c 2024-12-03T05:15:51,164 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d7c8a4ae9a5643a8a4d34d5ba494e72c, entries=100, sequenceid=710, filesize=9.6 K 2024-12-03T05:15:51,165 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b7e2b9d2e05056c58426b072bf89a451 in 938ms, sequenceid=710, compaction requested=true 2024-12-03T05:15:51,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e] to archive 2024-12-03T05:15:51,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:51,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/01a27813bf4d46c69d8b5b6bdf169cfc 2024-12-03T05:15:51,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/10d59a11ed2842bcaaf50b2d978d0d65 2024-12-03T05:15:51,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/8803896f340d487d872e36245f1cf5fe 2024-12-03T05:15:51,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d76774d05444414eaf6f26afbcaf9bac 2024-12-03T05:15:51,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/c857001305b948a09e9059d6df19060d 2024-12-03T05:15:51,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/62886b0d171c44b798112f4c8a9bb989 2024-12-03T05:15:51,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/47ce868ec1724b2a88364d2b15f3fb7f 2024-12-03T05:15:51,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4ce15a074604406a85802ce69ffd892d 2024-12-03T05:15:51,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/adfb8e4281cf4b96b272ecea1383f6d2 2024-12-03T05:15:51,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/db070c21a6204621b0248940fbb66ac9 2024-12-03T05:15:51,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/1a0a4717e8cb4f49837ed498695897ec 2024-12-03T05:15:51,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/a4abc63fa53147c9b3c8f71236c18342 2024-12-03T05:15:51,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/85df70946b4e457baf227e0d1ef469da 2024-12-03T05:15:51,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/7cc8907b09654de9be439a9e7fb2fdf4 2024-12-03T05:15:51,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f8aa1918bb104fa2bce3ac35979e628d 2024-12-03T05:15:51,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/83adafbbbb164242907a98e3675c464c 2024-12-03T05:15:51,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/4635d087453a4aaab33bf4f5f3b4a700 2024-12-03T05:15:51,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f6431376e12f444082529e7b41ae3795 2024-12-03T05:15:51,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/19b47e42098d4b0a8e1d7da3f8490a74 2024-12-03T05:15:51,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f667aa06b834409f92d75486507ed583 2024-12-03T05:15:51,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/776fa6a9f44f494392c2d83f67a5c56e 2024-12-03T05:15:51,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b4dae4176bb7413fbfcf4f6c124437f4 2024-12-03T05:15:51,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/e42fde2efab3410298f66fb7aa2e603e 2024-12-03T05:15:51,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/d4a18e2d7dab438cb541be10750f905f 2024-12-03T05:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:51,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/977d064e372742078d999c6e4868a8e1 2024-12-03T05:15:51,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/3b8c7074f51743b381da65180cbdeffb 2024-12-03T05:15:51,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/77971d0adc5c4b75b4ecc4fe1b89f0a6 2024-12-03T05:15:51,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/be7c1fd3c1d440779f90f4bc1b9f8d19 2024-12-03T05:15:51,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/f71be2b364d14d40bae9cd1ad1c79b2e 2024-12-03T05:15:51,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/88b0aa9b8d2b4fe3b1d107b982aae764, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d47cf308260c4591b03c1e462efaa2e6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/9f030801052c48ddbcd50e8881936a3b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d4d1e435a80d423d9bc20519702fbb88, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08839076ae3c438e9f2545e807c7c19f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/ba97b349b0724c82a37490280f59bcef, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b] to archive 2024-12-03T05:15:51,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:51,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/469d3d08263f46f1958795187c92bf1c 2024-12-03T05:15:51,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a19d3fd80a747cfb351cf4ac40421e7 2024-12-03T05:15:51,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/61a0550150dc4a4dacf65141f27e40e9 2024-12-03T05:15:51,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/88b0aa9b8d2b4fe3b1d107b982aae764 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/88b0aa9b8d2b4fe3b1d107b982aae764 2024-12-03T05:15:51,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/da425da4b7ea46a0a9f934545e7a4f72 2024-12-03T05:15:51,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/4a20d3b5a3b6465984c14bbf382fff44 2024-12-03T05:15:51,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d47cf308260c4591b03c1e462efaa2e6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d47cf308260c4591b03c1e462efaa2e6 2024-12-03T05:15:51,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/98a9a34093b04e3482f462f1eaa71747 2024-12-03T05:15:51,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/1651e51235c843e0907dd4c539360874 2024-12-03T05:15:51,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/9f030801052c48ddbcd50e8881936a3b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/9f030801052c48ddbcd50e8881936a3b 2024-12-03T05:15:51,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/25ee3ba66bd54975acd3a1e4a65c189f 2024-12-03T05:15:51,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0537cbc084904a17acb2b5cd88f799ef 2024-12-03T05:15:51,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d4d1e435a80d423d9bc20519702fbb88 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/d4d1e435a80d423d9bc20519702fbb88 2024-12-03T05:15:51,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/2a3749b75f0547e3a5a60ce220570229 2024-12-03T05:15:51,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/cc62b9985d8c4adea401a4277ce27197 2024-12-03T05:15:51,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/029aca683543424fb169f7e9af8a7fd0 2024-12-03T05:15:51,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08839076ae3c438e9f2545e807c7c19f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08839076ae3c438e9f2545e807c7c19f 2024-12-03T05:15:51,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/7bf8e0904b9a41c485e10e08d677c63e 2024-12-03T05:15:51,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/53f7721c71cb4bb2adc9978033fb5455 2024-12-03T05:15:51,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/ba97b349b0724c82a37490280f59bcef to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/ba97b349b0724c82a37490280f59bcef 2024-12-03T05:15:51,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/15e02c0025ba48709f5c5f305967f38c 2024-12-03T05:15:51,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/5952c22328294d989fe8b9b66ff5ba9d 2024-12-03T05:15:51,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/0f7d806fb87b4c1fb05dc85c3d1a1c1b 2024-12-03T05:15:51,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/86c9475da24f485a84dcbf56e4f23f8c 2024-12-03T05:15:51,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/43f12732787549168bafcdb3e7fed6d5 2024-12-03T05:15:51,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/005e0e1bba4a4b2d9f8d11a9d6b91b8b 2024-12-03T05:15:51,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/6e0196a26e8b4bbb9ff27dba57bf2c22 2024-12-03T05:15:51,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/bef14439e1d641c787f1a527322f6784 2024-12-03T05:15:51,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/012ea60f2b214daeb469009014d2c42b 2024-12-03T05:15:51,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b89ca4224794349acd10db946b94b66, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3adcfe3d26834e73a71e2886e32c33c4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f408d741ae604658b25e292b81a0ca1b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e48845cb2fdb430fb6e2350b05ba8dfa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e5f849b748dd4995aa43b68a3acff47c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/458387ae29254123a18c5400ec155a38, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f8bd5c21682c4d43a57536bbd4c33c3a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0] to archive 2024-12-03T05:15:51,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:15:51,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/bef1de9b3f824f85b3fe3897865ea326 2024-12-03T05:15:51,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ccb3c61f13584585a25a5f56700b6bf8 2024-12-03T05:15:51,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d0cd4a45bbad426d885561bf81973b57 2024-12-03T05:15:51,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e534823b05c48b79bb75cda81bdd5b4 2024-12-03T05:15:51,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b89ca4224794349acd10db946b94b66 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b89ca4224794349acd10db946b94b66 2024-12-03T05:15:51,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/80ee10a3d10e4adcb3c95581d623237a 2024-12-03T05:15:51,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0777b87058194aa89a569c15771d2cee 2024-12-03T05:15:51,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b425c63d5ef94486b8b26ecc7313e087 2024-12-03T05:15:51,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3adcfe3d26834e73a71e2886e32c33c4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3adcfe3d26834e73a71e2886e32c33c4 2024-12-03T05:15:51,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/0e72600ffe9a4658a5ee8f0c109944f7 2024-12-03T05:15:51,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/b5e6cf43e38e42b2ac74087cf94f8d87 2024-12-03T05:15:51,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f408d741ae604658b25e292b81a0ca1b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f408d741ae604658b25e292b81a0ca1b 2024-12-03T05:15:51,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/2e105cdcd8404e7aa60f327d4e27438b 2024-12-03T05:15:51,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/3373d5bc80e245e1aef3b98eb0f2e447 2024-12-03T05:15:51,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/7b49eca4b1624b369750aaec15bf9d38 2024-12-03T05:15:51,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e48845cb2fdb430fb6e2350b05ba8dfa to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e48845cb2fdb430fb6e2350b05ba8dfa 2024-12-03T05:15:51,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/af1f3de6b00a4f37a8753fc5b560ff08 2024-12-03T05:15:51,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/ce48b3a192d444d6931cb5867664e63b 2024-12-03T05:15:51,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e5f849b748dd4995aa43b68a3acff47c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/e5f849b748dd4995aa43b68a3acff47c 2024-12-03T05:15:51,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/508804504c254f7491d2d53499d4c298 2024-12-03T05:15:51,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/1707455131f84c298a268aa24876a947 2024-12-03T05:15:51,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/458387ae29254123a18c5400ec155a38 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/458387ae29254123a18c5400ec155a38 2024-12-03T05:15:51,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/9c7a24d3a79c4a529d404b86c2b21439 2024-12-03T05:15:51,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/8fc7ae36f2e4478bb2ff1f74f72066a8 2024-12-03T05:15:51,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f8bd5c21682c4d43a57536bbd4c33c3a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f8bd5c21682c4d43a57536bbd4c33c3a 2024-12-03T05:15:51,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/29566c6ff2b8421da59bc71ccc876139 2024-12-03T05:15:51,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/c63c8b9d33044c0484f5fa8eb58d14d6 2024-12-03T05:15:51,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/350e89fff41047e0a9e8392e4c8fdee0 2024-12-03T05:15:51,279 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits/713.seqid, newMaxSeqId=713, maxSeqId=4 2024-12-03T05:15:51,280 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. 2024-12-03T05:15:51,280 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for b7e2b9d2e05056c58426b072bf89a451: Waiting for close lock at 1733202950227Running coprocessor pre-close hooks at 1733202950227Disabling compacts and flushes for region at 1733202950227Disabling writes for close at 1733202950227Obtaining lock to block concurrent updates at 1733202950227Preparing flush snapshotting stores in b7e2b9d2e05056c58426b072bf89a451 at 1733202950227Finished memstore snapshotting TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733202950227Flushing stores of TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451. at 1733202950228 (+1 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/A: creating writer at 1733202950228Flushing mob file b7e2b9d2e05056c58426b072bf89a451/A: appending metadata at 1733202950233 (+5 ms)Flushing mob file b7e2b9d2e05056c58426b072bf89a451/A: closing flushed file at 1733202950233Flushing b7e2b9d2e05056c58426b072bf89a451/A: appending metadata at 1733202950643 (+410 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/A: closing flushed file at 1733202950643Flushing b7e2b9d2e05056c58426b072bf89a451/B: creating writer at 1733202951063 (+420 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/B: appending metadata at 1733202951069 (+6 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/B: closing flushed file at 1733202951069Flushing b7e2b9d2e05056c58426b072bf89a451/C: creating writer at 1733202951105 (+36 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/C: appending metadata at 1733202951121 (+16 ms)Flushing b7e2b9d2e05056c58426b072bf89a451/C: closing flushed file at 1733202951121Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f02cb37: reopening flushed file at 1733202951140 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@273d4f21: reopening flushed file at 1733202951150 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63e6e888: reopening flushed file at 1733202951158 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b7e2b9d2e05056c58426b072bf89a451 in 938ms, sequenceid=710, compaction requested=true at 1733202951165 (+7 ms)Writing region close event to WAL at 1733202951277 (+112 ms)Running coprocessor post-close hooks at 1733202951280 (+3 ms)Closed at 1733202951280 2024-12-03T05:15:51,282 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:51,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b7e2b9d2e05056c58426b072bf89a451, regionState=CLOSED 2024-12-03T05:15:51,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:51,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T05:15:51,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure b7e2b9d2e05056c58426b072bf89a451, server=e3c6570475b6,37863,1733202887099 in 1.2100 sec 2024-12-03T05:15:51,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-03T05:15:51,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7e2b9d2e05056c58426b072bf89a451, UNASSIGN in 1.2140 sec 2024-12-03T05:15:51,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T05:15:51,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2180 sec 2024-12-03T05:15:51,289 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202951289"}]},"ts":"1733202951289"} 2024-12-03T05:15:51,290 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:15:51,290 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:15:51,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2270 sec 2024-12-03T05:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T05:15:52,211 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:52,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,213 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,214 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T05:15:52,216 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,219 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits] 2024-12-03T05:15:52,221 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/0700331340544c58803eb13d6168a21d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/0700331340544c58803eb13d6168a21d 2024-12-03T05:15:52,223 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/22af270daed24052b4a29f4b7782e4eb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/22af270daed24052b4a29f4b7782e4eb 2024-12-03T05:15:52,226 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/5769532c9f16494e9065f2de3355f71a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/5769532c9f16494e9065f2de3355f71a 2024-12-03T05:15:52,227 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b70576d5f9534a16a27b4221bf8091fe to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/A/b70576d5f9534a16a27b4221bf8091fe 2024-12-03T05:15:52,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08467f0ab6e647f6a1db6f270b9c307a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/08467f0ab6e647f6a1db6f270b9c307a 2024-12-03T05:15:52,232 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/625503ad753145f5b32465fc647bfd3a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/625503ad753145f5b32465fc647bfd3a 2024-12-03T05:15:52,234 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/97f2b43f70de4795980d8cd6e74ccc1c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/97f2b43f70de4795980d8cd6e74ccc1c 2024-12-03T05:15:52,236 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/c38d8e223f5443b2a5062178a25bad00 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/B/c38d8e223f5443b2a5062178a25bad00 2024-12-03T05:15:52,238 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/61c6307b9c4542d9a2affbfff99eb9b2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/61c6307b9c4542d9a2affbfff99eb9b2 2024-12-03T05:15:52,241 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/6c32f6806faf49fd88fb43450f3c0332 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/6c32f6806faf49fd88fb43450f3c0332 2024-12-03T05:15:52,243 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d7c8a4ae9a5643a8a4d34d5ba494e72c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/d7c8a4ae9a5643a8a4d34d5ba494e72c 2024-12-03T05:15:52,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f9710230e75c42df83c41234287c4767 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/C/f9710230e75c42df83c41234287c4767 2024-12-03T05:15:52,249 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits/713.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451/recovered.edits/713.seqid 2024-12-03T05:15:52,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:15:52,250 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:15:52,251 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T05:15:52,257 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120300a6909122fa43d49b93e6c936f2cc59_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120300a6909122fa43d49b93e6c936f2cc59_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,258 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030dd2777364314d9e9ac962bd9c8e42ad_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030dd2777364314d9e9ac962bd9c8e42ad_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,261 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203210d0634a4b643fe87c1e266f8df28e9_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203210d0634a4b643fe87c1e266f8df28e9_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,262 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203368f6b3bd5104798ad6b42adaf599f3e_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203368f6b3bd5104798ad6b42adaf599f3e_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,264 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350648a02068c4cdd94e19fbdaad17c71_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350648a02068c4cdd94e19fbdaad17c71_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,265 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203568c5f15817246ac8b1eb177306c4daf_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203568c5f15817246ac8b1eb177306c4daf_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,267 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120365cf5e8df59e4a0ea631e4ca213be84b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120365cf5e8df59e4a0ea631e4ca213be84b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,268 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203678d89448dc74d9aa88f28643b7deefd_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203678d89448dc74d9aa88f28643b7deefd_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,270 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388765c167ce34ad194334f162bd95490_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388765c167ce34ad194334f162bd95490_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,271 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391097e4048f54cd3b4c010f5c2bb0d69_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391097e4048f54cd3b4c010f5c2bb0d69_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,273 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395be59aa74de441c8247924141df866b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395be59aa74de441c8247924141df866b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,275 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203978f59a0e84540f59c5acd2b1057188d_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203978f59a0e84540f59c5acd2b1057188d_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,277 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120397af45f239af4443a18ae303e6633dd2_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120397af45f239af4443a18ae303e6633dd2_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,279 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a54489b0ae2646b78b852dbefe2afe80_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a54489b0ae2646b78b852dbefe2afe80_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,289 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a78c71ad66e342d6954a7a4810ce6e24_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a78c71ad66e342d6954a7a4810ce6e24_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad7de245002045e8a9244e3853817acf_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad7de245002045e8a9244e3853817acf_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beb104aa04654f959483a739173283d4_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beb104aa04654f959483a739173283d4_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cad3927c52ea46be93d29cf9fab9694e_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cad3927c52ea46be93d29cf9fab9694e_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T05:15:52,327 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce30099f6d5241db892ba3f22b6a6131_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce30099f6d5241db892ba3f22b6a6131_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,329 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d09462ddf0304c0bbb9d1073a3263a9b_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d09462ddf0304c0bbb9d1073a3263a9b_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,337 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbf0b687156f4522b30a76f66bb71b66_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbf0b687156f4522b30a76f66bb71b66_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,339 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dead705e6c8f4d36bef3c23f5562f65d_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dead705e6c8f4d36bef3c23f5562f65d_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,343 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed19a09e99f74ecc99ec1d0c1e6e5b48_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed19a09e99f74ecc99ec1d0c1e6e5b48_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,345 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff46147a588c4996b80c556f424628fe_b7e2b9d2e05056c58426b072bf89a451 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff46147a588c4996b80c556f424628fe_b7e2b9d2e05056c58426b072bf89a451 2024-12-03T05:15:52,347 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:15:52,352 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,356 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:15:52,369 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:15:52,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:15:52,377 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202952377"}]},"ts":"9223372036854775807"} 2024-12-03T05:15:52,384 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:15:52,384 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b7e2b9d2e05056c58426b072bf89a451, NAME => 'TestAcidGuarantees,,1733202919183.b7e2b9d2e05056c58426b072bf89a451.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:15:52,385 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:15:52,385 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202952385"}]},"ts":"9223372036854775807"} 2024-12-03T05:15:52,400 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:15:52,409 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 197 msec 2024-12-03T05:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T05:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:15:52,531 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:52,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-443467210_22 at /127.0.0.1:49428 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=714 (was 719), ProcessCount=11 (was 11), AvailableMemoryMB=2898 (was 3835) 2024-12-03T05:15:52,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=714, ProcessCount=11, AvailableMemoryMB=2897 2024-12-03T05:15:52,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:15:52,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:15:52,557 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:15:52,557 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:52,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-03T05:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:52,558 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:15:52,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742081_1257 (size=1061) 2024-12-03T05:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:52,975 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:15:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742082_1258 (size=53) 2024-12-03T05:15:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 337aee72174b5cc395d13e96dd50849b, disabling compactions & flushes 2024-12-03T05:15:53,383 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. after waiting 0 ms 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,383 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,383 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 337aee72174b5cc395d13e96dd50849b: Waiting for close lock at 1733202953383Disabling compacts and flushes for region at 1733202953383Disabling writes for close at 1733202953383Writing region close event to WAL at 1733202953383Closed at 1733202953383 2024-12-03T05:15:53,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:15:53,384 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202953384"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202953384"}]},"ts":"1733202953384"} 2024-12-03T05:15:53,385 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:15:53,386 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:15:53,386 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202953386"}]},"ts":"1733202953386"} 2024-12-03T05:15:53,387 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:15:53,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, ASSIGN}] 2024-12-03T05:15:53,389 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, ASSIGN 2024-12-03T05:15:53,390 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:15:53,540 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=337aee72174b5cc395d13e96dd50849b, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:53,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, ASSIGN because future has completed 2024-12-03T05:15:53,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:15:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:53,698 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,698 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:15:53,698 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,698 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:15:53,698 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,698 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,700 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,705 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:53,705 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 337aee72174b5cc395d13e96dd50849b columnFamilyName A 2024-12-03T05:15:53,705 DEBUG [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:53,707 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(327): Store=337aee72174b5cc395d13e96dd50849b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:53,707 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,710 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:53,710 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 337aee72174b5cc395d13e96dd50849b columnFamilyName B 2024-12-03T05:15:53,710 DEBUG [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:53,712 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(327): Store=337aee72174b5cc395d13e96dd50849b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:53,712 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,713 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:15:53,713 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 337aee72174b5cc395d13e96dd50849b columnFamilyName C 2024-12-03T05:15:53,713 DEBUG [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:15:53,714 INFO [StoreOpener-337aee72174b5cc395d13e96dd50849b-1 {}] regionserver.HStore(327): Store=337aee72174b5cc395d13e96dd50849b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:15:53,715 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,715 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,720 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,721 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,722 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,722 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,722 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:15:53,724 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,726 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:15:53,727 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 337aee72174b5cc395d13e96dd50849b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62206723, jitterRate=-0.07304759323596954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:15:53,727 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:53,727 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 337aee72174b5cc395d13e96dd50849b: Running coprocessor pre-open hook at 1733202953699Writing region info on filesystem at 1733202953699Initializing all the Stores at 1733202953700 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202953700Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202953700Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202953700Cleaning up temporary data from old regions at 1733202953722 (+22 ms)Running coprocessor post-open hooks at 1733202953727 (+5 ms)Region opened successfully at 1733202953727 2024-12-03T05:15:53,728 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., pid=61, masterSystemTime=1733202953695 2024-12-03T05:15:53,730 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,730 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:53,731 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=337aee72174b5cc395d13e96dd50849b, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:53,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:15:53,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T05:15:53,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 in 191 msec 2024-12-03T05:15:53,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-03T05:15:53,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, ASSIGN in 348 msec 2024-12-03T05:15:53,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:15:53,739 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202953739"}]},"ts":"1733202953739"} 2024-12-03T05:15:53,741 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:15:53,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:15:53,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-12-03T05:15:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T05:15:54,711 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:54,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c08f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56035544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfa211e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:15:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:15:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3467c31a 2024-12-03T05:15:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:15:54,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:15:54,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd8e12d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35562eaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f38c060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429799b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd7f88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480421b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682e3aa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffe145, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6327b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@101bcf78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390173a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fdf952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64fa4341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78813486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0cbf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,742 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f094dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6326466f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,745 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622e8c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334ea8bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212d94ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,749 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b84803b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,751 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db8b3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bafd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,753 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747683d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77df1bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b15c907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:15:54,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:15:54,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:15:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:15:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:15:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:15:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:15:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:15:54,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:15:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f5d75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:15:54,758 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,758 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,758 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,758 DEBUG [Thread-1208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,759 DEBUG [Thread-1210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,760 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,762 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,770 DEBUG [Thread-1199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,771 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,772 DEBUG [Thread-1206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:15:54,772 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:15:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:15:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-03T05:15:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:54,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:54,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:54,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:54,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:15:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:15:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:54,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:54,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/81e50ec8c45c4b4c91bf0580c2ad8896 is 50, key is test_row_0/A:col10/1733202954778/Put/seqid=0 2024-12-03T05:15:54,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202954829, max timestamp=1733202954847Num uniques 859; 2024-12-03T05:15:54,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:54,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202954829, max timestamp=1733202954847Num uniques 859; 2024-12-03T05:15:54,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:54,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202954829, max timestamp=1733202954847Num uniques 859; 2024-12-03T05:15:54,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:15:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742083_1259 (size=9657) 2024-12-03T05:15:54,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/81e50ec8c45c4b4c91bf0580c2ad8896 2024-12-03T05:15:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203014903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203014904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203014901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:54,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203014918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:54,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203014919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:54,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:15:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:54,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:54,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/d131561e9f93471986ee73c5dd41ccd8 is 50, key is test_row_0/B:col10/1733202954778/Put/seqid=0 2024-12-03T05:15:55,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742084_1260 (size=9657) 2024-12-03T05:15:55,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/d131561e9f93471986ee73c5dd41ccd8 2024-12-03T05:15:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203015030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203015030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203015030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203015041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203015041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/777f53f734a44622a63fcce4fb01d2ac is 50, key is test_row_0/C:col10/1733202954778/Put/seqid=0 2024-12-03T05:15:55,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742085_1261 (size=9657) 2024-12-03T05:15:55,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:15:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:55,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:15:55,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203015240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203015241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203015241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203015250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203015250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:15:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:55,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/777f53f734a44622a63fcce4fb01d2ac 2024-12-03T05:15:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/81e50ec8c45c4b4c91bf0580c2ad8896 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896 2024-12-03T05:15:55,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896, entries=100, sequenceid=12, filesize=9.4 K 2024-12-03T05:15:55,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/d131561e9f93471986ee73c5dd41ccd8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8 2024-12-03T05:15:55,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8, entries=100, sequenceid=12, filesize=9.4 K 2024-12-03T05:15:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/777f53f734a44622a63fcce4fb01d2ac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac 2024-12-03T05:15:55,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac, entries=100, sequenceid=12, filesize=9.4 K 2024-12-03T05:15:55,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 337aee72174b5cc395d13e96dd50849b in 747ms, sequenceid=12, compaction requested=false 2024-12-03T05:15:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:55,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:55,549 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:15:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/637e7c0abedd44a98118950ce68fb7cb is 50, key is test_row_0/A:col10/1733202954913/Put/seqid=0 2024-12-03T05:15:55,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203015563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742086_1262 (size=12001) 2024-12-03T05:15:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203015564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203015566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203015567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203015567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203015670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203015680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203015680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203015680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203015880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203015890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203015890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203015892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:55,968 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/637e7c0abedd44a98118950ce68fb7cb 2024-12-03T05:15:56,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 is 50, key is test_row_0/B:col10/1733202954913/Put/seqid=0 2024-12-03T05:15:56,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203016071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742087_1263 (size=12001) 2024-12-03T05:15:56,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 2024-12-03T05:15:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/eb9eb1e0a26a4e549a025cae7f2afaf9 is 50, key is test_row_0/C:col10/1733202954913/Put/seqid=0 2024-12-03T05:15:56,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742088_1264 (size=12001) 2024-12-03T05:15:56,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203016197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203016201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203016201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203016201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,208 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/eb9eb1e0a26a4e549a025cae7f2afaf9 2024-12-03T05:15:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/637e7c0abedd44a98118950ce68fb7cb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb 2024-12-03T05:15:56,232 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:15:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 2024-12-03T05:15:56,238 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:15:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/eb9eb1e0a26a4e549a025cae7f2afaf9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9 2024-12-03T05:15:56,249 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T05:15:56,250 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 337aee72174b5cc395d13e96dd50849b in 701ms, sequenceid=48, compaction requested=false 2024-12-03T05:15:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T05:15:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T05:15:56,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T05:15:56,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-12-03T05:15:56,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.4840 sec 2024-12-03T05:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:56,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T05:15:56,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:56,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:56,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2888bf411b76452e9f20d1c21cedf404 is 50, key is test_row_0/A:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:56,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742089_1265 (size=12001) 2024-12-03T05:15:56,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2888bf411b76452e9f20d1c21cedf404 2024-12-03T05:15:56,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203016756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203016759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203016762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203016763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/58a0f82943fd49688fdcf60ac6570ec7 is 50, key is test_row_0/B:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:56,825 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:15:56,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742090_1266 (size=12001) 2024-12-03T05:15:56,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/58a0f82943fd49688fdcf60ac6570ec7 2024-12-03T05:15:56,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203016871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203016872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203016872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203016881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:56,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9995cd59ea684c448cf516556711636a is 50, key is test_row_0/C:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742091_1267 (size=12001) 2024-12-03T05:15:56,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9995cd59ea684c448cf516556711636a 2024-12-03T05:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T05:15:56,921 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2888bf411b76452e9f20d1c21cedf404 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404 2024-12-03T05:15:56,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-03T05:15:56,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:15:56,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:56,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:56,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T05:15:56,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/58a0f82943fd49688fdcf60ac6570ec7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7 2024-12-03T05:15:56,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T05:15:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9995cd59ea684c448cf516556711636a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a 2024-12-03T05:15:56,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T05:15:56,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 337aee72174b5cc395d13e96dd50849b in 248ms, sequenceid=62, compaction requested=true 2024-12-03T05:15:56,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:15:56,973 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:56,973 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:15:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:56,974 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:56,974 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:56,975 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:15:56,975 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:15:56,975 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:56,975 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:56,975 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=32.9 K 2024-12-03T05:15:56,975 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=32.9 K 2024-12-03T05:15:56,976 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d131561e9f93471986ee73c5dd41ccd8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202954778 2024-12-03T05:15:56,976 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e50ec8c45c4b4c91bf0580c2ad8896, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202954778 2024-12-03T05:15:56,977 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6c0f52f45943d19bec7d7d4dc6f2c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202954887 2024-12-03T05:15:56,977 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 637e7c0abedd44a98118950ce68fb7cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202954887 2024-12-03T05:15:56,977 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 58a0f82943fd49688fdcf60ac6570ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:15:56,978 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2888bf411b76452e9f20d1c21cedf404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:15:56,996 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:56,996 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/10fb3944faf84a5d930d3685ea3dfb0f is 50, key is test_row_0/A:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:57,001 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:57,001 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e579fa2d4ae94cb2a3dc6784c56d70f8 is 50, key is test_row_0/B:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:57,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742093_1269 (size=12104) 2024-12-03T05:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:15:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742092_1268 (size=12104) 2024-12-03T05:15:57,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T05:15:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:57,087 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T05:15:57,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:57,098 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e579fa2d4ae94cb2a3dc6784c56d70f8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e579fa2d4ae94cb2a3dc6784c56d70f8 2024-12-03T05:15:57,105 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into e579fa2d4ae94cb2a3dc6784c56d70f8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:57,105 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:57,105 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=13, startTime=1733202956973; duration=0sec 2024-12-03T05:15:57,105 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:15:57,105 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:15:57,105 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:15:57,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/baede85b05dd4369852e16f27fb7a3a3 is 50, key is test_row_0/A:col10/1733202956761/Put/seqid=0 2024-12-03T05:15:57,109 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:15:57,109 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:15:57,109 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:57,110 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=32.9 K 2024-12-03T05:15:57,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 777f53f734a44622a63fcce4fb01d2ac, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733202954778 2024-12-03T05:15:57,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting eb9eb1e0a26a4e549a025cae7f2afaf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733202954887 2024-12-03T05:15:57,112 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9995cd59ea684c448cf516556711636a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:15:57,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203017121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203017125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203017125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742094_1270 (size=14341) 2024-12-03T05:15:57,131 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/baede85b05dd4369852e16f27fb7a3a3 2024-12-03T05:15:57,134 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:15:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/d86f18bb866a411f877b2ce0c7a09296 is 50, key is test_row_0/C:col10/1733202956713/Put/seqid=0 2024-12-03T05:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203017129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203017133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/04a533209d964bb8a484ef1ce3988f9b is 50, key is test_row_0/B:col10/1733202956761/Put/seqid=0 2024-12-03T05:15:57,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742095_1271 (size=12104) 2024-12-03T05:15:57,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742096_1272 (size=12001) 2024-12-03T05:15:57,235 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/04a533209d964bb8a484ef1ce3988f9b 2024-12-03T05:15:57,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203017231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203017241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203017241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203017245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203017252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/06cc3f785eb5407db8311f7f3112812f is 50, key is test_row_0/C:col10/1733202956761/Put/seqid=0 2024-12-03T05:15:57,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742097_1273 (size=12001) 2024-12-03T05:15:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203017442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,458 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/10fb3944faf84a5d930d3685ea3dfb0f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/10fb3944faf84a5d930d3685ea3dfb0f 2024-12-03T05:15:57,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203017451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,465 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into 10fb3944faf84a5d930d3685ea3dfb0f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:57,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:57,466 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=13, startTime=1733202956973; duration=0sec 2024-12-03T05:15:57,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:57,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:15:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203017460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203017461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203017461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:15:57,608 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/d86f18bb866a411f877b2ce0c7a09296 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/d86f18bb866a411f877b2ce0c7a09296 2024-12-03T05:15:57,626 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into d86f18bb866a411f877b2ce0c7a09296(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:15:57,626 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:57,626 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=13, startTime=1733202956973; duration=0sec 2024-12-03T05:15:57,626 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:15:57,626 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:15:57,762 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/06cc3f785eb5407db8311f7f3112812f 2024-12-03T05:15:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203017764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203017771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203017781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203017782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203017784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:57,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/baede85b05dd4369852e16f27fb7a3a3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3 2024-12-03T05:15:57,813 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3, entries=200, sequenceid=81, filesize=14.0 K 2024-12-03T05:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/04a533209d964bb8a484ef1ce3988f9b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b 2024-12-03T05:15:57,859 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b, entries=150, sequenceid=81, filesize=11.7 K 2024-12-03T05:15:57,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/06cc3f785eb5407db8311f7f3112812f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f 2024-12-03T05:15:57,875 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f, entries=150, sequenceid=81, filesize=11.7 K 2024-12-03T05:15:57,882 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 337aee72174b5cc395d13e96dd50849b in 795ms, sequenceid=81, compaction requested=false 2024-12-03T05:15:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-03T05:15:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-03T05:15:57,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T05:15:57,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 958 msec 2024-12-03T05:15:57,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 981 msec 2024-12-03T05:15:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T05:15:58,071 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:58,071 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T05:15:58,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:15:58,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:58,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:15:58,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T05:15:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:58,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:58,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/c59d8fc8d7dc4113bbd324709fc15d6b is 50, key is test_row_0/A:col10/1733202957116/Put/seqid=0 2024-12-03T05:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:58,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:58,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742098_1274 (size=12001) 2024-12-03T05:15:58,299 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/c59d8fc8d7dc4113bbd324709fc15d6b 2024-12-03T05:15:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2c9c166f1e764cce9ed2602ce0f6f6e2 is 50, key is test_row_0/B:col10/1733202957116/Put/seqid=0 2024-12-03T05:15:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203018317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203018318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203018318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203018323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203018325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742099_1275 (size=12001) 2024-12-03T05:15:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:15:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203018430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203018431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203018431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203018431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203018446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203018641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203018641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203018642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203018643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203018664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:15:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2c9c166f1e764cce9ed2602ce0f6f6e2 2024-12-03T05:15:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/4fe939ae0cb44b17b50ca15639c3ef6d is 50, key is test_row_0/C:col10/1733202957116/Put/seqid=0 2024-12-03T05:15:58,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742100_1276 (size=12001) 2024-12-03T05:15:58,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/4fe939ae0cb44b17b50ca15639c3ef6d 2024-12-03T05:15:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/c59d8fc8d7dc4113bbd324709fc15d6b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b 2024-12-03T05:15:58,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T05:15:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2c9c166f1e764cce9ed2602ce0f6f6e2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2 2024-12-03T05:15:58,931 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T05:15:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/4fe939ae0cb44b17b50ca15639c3ef6d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d 2024-12-03T05:15:58,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203018952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203018952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203018952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203018962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:58,977 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T05:15:58,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 337aee72174b5cc395d13e96dd50849b in 734ms, sequenceid=98, compaction requested=true 2024-12-03T05:15:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:15:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T05:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T05:15:58,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T05:15:58,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 901 msec 2024-12-03T05:15:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:15:59,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:15:59,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:15:59,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 917 msec 2024-12-03T05:15:59,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e0877ffb8b0477380480054455666a0 is 50, key is test_row_0/A:col10/1733202958989/Put/seqid=0 2024-12-03T05:15:59,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742101_1277 (size=12001) 2024-12-03T05:15:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203019124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T05:15:59,231 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:15:59,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T05:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:15:59,236 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:15:59,238 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:15:59,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:15:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203019240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:15:59,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:15:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203019460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203019460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203019470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203019471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203019473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e0877ffb8b0477380480054455666a0 2024-12-03T05:15:59,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/371c8a2c3ccd4e53834931662a21ee9d is 50, key is test_row_0/B:col10/1733202958989/Put/seqid=0 2024-12-03T05:15:59,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742102_1278 (size=12001) 2024-12-03T05:15:59,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:15:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:15:59,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:15:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:15:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203019772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:15:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:15:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:15:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:15:59,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:15:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:15:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:15:59,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:15:59,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/371c8a2c3ccd4e53834931662a21ee9d 2024-12-03T05:15:59,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/23fd4a314e394707befa404851ce2718 is 50, key is test_row_0/C:col10/1733202958989/Put/seqid=0 2024-12-03T05:15:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742103_1279 (size=12001) 2024-12-03T05:16:00,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/23fd4a314e394707befa404851ce2718 2024-12-03T05:16:00,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e0877ffb8b0477380480054455666a0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0 2024-12-03T05:16:00,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:16:00,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:00,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:00,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:00,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T05:16:00,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/371c8a2c3ccd4e53834931662a21ee9d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d 2024-12-03T05:16:00,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T05:16:00,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/23fd4a314e394707befa404851ce2718 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718 2024-12-03T05:16:00,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T05:16:00,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 337aee72174b5cc395d13e96dd50849b in 1104ms, sequenceid=117, compaction requested=true 2024-12-03T05:16:00,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:00,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:00,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:00,104 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:00,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:00,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:16:00,106 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:00,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:00,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:00,111 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,112 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/10fb3944faf84a5d930d3685ea3dfb0f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=49.3 K 2024-12-03T05:16:00,114 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10fb3944faf84a5d930d3685ea3dfb0f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:16:00,115 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting baede85b05dd4369852e16f27fb7a3a3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733202956756 2024-12-03T05:16:00,119 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59d8fc8d7dc4113bbd324709fc15d6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733202957116 2024-12-03T05:16:00,120 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0877ffb8b0477380480054455666a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:00,124 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:00,124 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:00,124 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,124 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e579fa2d4ae94cb2a3dc6784c56d70f8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=47.0 K 2024-12-03T05:16:00,126 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e579fa2d4ae94cb2a3dc6784c56d70f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:16:00,129 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a533209d964bb8a484ef1ce3988f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733202956759 2024-12-03T05:16:00,130 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9c166f1e764cce9ed2602ce0f6f6e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733202957116 2024-12-03T05:16:00,131 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 371c8a2c3ccd4e53834931662a21ee9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:00,158 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:00,158 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:00,159 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/84daa655f9bf4149aad238ac46a08ab3 is 50, key is test_row_0/A:col10/1733202958989/Put/seqid=0 2024-12-03T05:16:00,159 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/740fb04f6f7541e4bdc882aa0ede79a0 is 50, key is test_row_0/B:col10/1733202958989/Put/seqid=0 2024-12-03T05:16:00,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742105_1281 (size=12241) 2024-12-03T05:16:00,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T05:16:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,217 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:00,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742104_1280 (size=12241) 2024-12-03T05:16:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0184c6a3dbfb41049f7e46399c94d3e7 is 50, key is test_row_0/A:col10/1733202959121/Put/seqid=0 2024-12-03T05:16:00,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742106_1282 (size=12051) 2024-12-03T05:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:00,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:16:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203020376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203020480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203020481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,485 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,485 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203020494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203020494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,496 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:00,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203020494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,500 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,623 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/740fb04f6f7541e4bdc882aa0ede79a0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/740fb04f6f7541e4bdc882aa0ede79a0 2024-12-03T05:16:00,630 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into 740fb04f6f7541e4bdc882aa0ede79a0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:00,630 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:00,630 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=12, startTime=1733202960104; duration=0sec 2024-12-03T05:16:00,630 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:00,630 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:00,630 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:00,635 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:00,635 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:00,635 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:00,635 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/d86f18bb866a411f877b2ce0c7a09296, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=47.0 K 2024-12-03T05:16:00,637 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d86f18bb866a411f877b2ce0c7a09296, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733202955556 2024-12-03T05:16:00,637 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cc3f785eb5407db8311f7f3112812f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733202956759 2024-12-03T05:16:00,645 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe939ae0cb44b17b50ca15639c3ef6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733202957116 2024-12-03T05:16:00,648 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fd4a314e394707befa404851ce2718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:00,657 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/84daa655f9bf4149aad238ac46a08ab3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/84daa655f9bf4149aad238ac46a08ab3 2024-12-03T05:16:00,665 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0184c6a3dbfb41049f7e46399c94d3e7 2024-12-03T05:16:00,709 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into 84daa655f9bf4149aad238ac46a08ab3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:00,710 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:00,710 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=12, startTime=1733202960104; duration=0sec 2024-12-03T05:16:00,710 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:00,710 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:00,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203020711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:00,731 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:00,733 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/531787a49e9d4003861c7a64e686e55c is 50, key is test_row_0/C:col10/1733202958989/Put/seqid=0 2024-12-03T05:16:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e09e16c291834353ac8e9d44b499e8b7 is 50, key is test_row_0/B:col10/1733202959121/Put/seqid=0 2024-12-03T05:16:00,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742107_1283 (size=12241) 2024-12-03T05:16:00,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742108_1284 (size=12051) 2024-12-03T05:16:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203021030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:01,187 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/531787a49e9d4003861c7a64e686e55c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/531787a49e9d4003861c7a64e686e55c 2024-12-03T05:16:01,201 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into 531787a49e9d4003861c7a64e686e55c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:01,201 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:01,201 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=12, startTime=1733202960104; duration=0sec 2024-12-03T05:16:01,201 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:01,201 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:01,232 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e09e16c291834353ac8e9d44b499e8b7 2024-12-03T05:16:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/bbfbacaec36842c38c8f36a666fd3a05 is 50, key is test_row_0/C:col10/1733202959121/Put/seqid=0 2024-12-03T05:16:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742109_1285 (size=12051) 2024-12-03T05:16:01,256 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/bbfbacaec36842c38c8f36a666fd3a05 2024-12-03T05:16:01,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0184c6a3dbfb41049f7e46399c94d3e7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7 2024-12-03T05:16:01,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T05:16:01,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e09e16c291834353ac8e9d44b499e8b7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7 2024-12-03T05:16:01,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T05:16:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/bbfbacaec36842c38c8f36a666fd3a05 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05 2024-12-03T05:16:01,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T05:16:01,327 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 1111ms, sequenceid=130, compaction requested=false 2024-12-03T05:16:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T05:16:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T05:16:01,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T05:16:01,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-12-03T05:16:01,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1030 sec 2024-12-03T05:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T05:16:01,381 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:01,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T05:16:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:16:01,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:01,394 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:01,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:16:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:01,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:16:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:01,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7b467c937e21413d966a5d23e4928cbe is 50, key is test_row_0/A:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:01,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742110_1286 (size=12151) 2024-12-03T05:16:01,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7b467c937e21413d966a5d23e4928cbe 2024-12-03T05:16:01,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/12862ba9803049d9a7898bb66d07b95d is 50, key is test_row_0/B:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:01,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742111_1287 (size=12151) 2024-12-03T05:16:01,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/12862ba9803049d9a7898bb66d07b95d 2024-12-03T05:16:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203021616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:01,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9922de70d2be4db29e1523acce6495ea is 50, key is test_row_0/C:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:01,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742112_1288 (size=12151) 2024-12-03T05:16:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:16:01,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:16:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203021730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:01,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:16:01,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:01,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:01,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:01,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203021940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:16:02,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:16:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9922de70d2be4db29e1523acce6495ea 2024-12-03T05:16:02,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7b467c937e21413d966a5d23e4928cbe as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe 2024-12-03T05:16:02,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T05:16:02,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/12862ba9803049d9a7898bb66d07b95d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d 2024-12-03T05:16:02,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T05:16:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/9922de70d2be4db29e1523acce6495ea as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea 2024-12-03T05:16:02,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T05:16:02,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 337aee72174b5cc395d13e96dd50849b in 560ms, sequenceid=153, compaction requested=true 2024-12-03T05:16:02,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:02,108 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:02,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:02,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:02,108 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:02,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:02,110 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:02,110 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:02,110 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,110 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:02,110 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:02,110 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/84daa655f9bf4149aad238ac46a08ab3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=35.6 K 2024-12-03T05:16:02,110 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,110 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/740fb04f6f7541e4bdc882aa0ede79a0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=35.6 K 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84daa655f9bf4149aad238ac46a08ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 740fb04f6f7541e4bdc882aa0ede79a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0184c6a3dbfb41049f7e46399c94d3e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733202959087 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e09e16c291834353ac8e9d44b499e8b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733202959087 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b467c937e21413d966a5d23e4928cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:02,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 12862ba9803049d9a7898bb66d07b95d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:02,127 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:02,127 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:02,128 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/9125bc7e11f34bc492acd7a23b0d093a is 50, key is test_row_0/A:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:02,128 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/fe748fe920af42309a09199afff5576b is 50, key is test_row_0/B:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742113_1289 (size=12493) 2024-12-03T05:16:02,138 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/fe748fe920af42309a09199afff5576b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fe748fe920af42309a09199afff5576b 2024-12-03T05:16:02,147 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into fe748fe920af42309a09199afff5576b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:02,147 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:02,147 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=13, startTime=1733202962108; duration=0sec 2024-12-03T05:16:02,147 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:02,147 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:02,147 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:02,150 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:02,150 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:02,150 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,150 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/531787a49e9d4003861c7a64e686e55c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=35.6 K 2024-12-03T05:16:02,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 531787a49e9d4003861c7a64e686e55c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733202958316 2024-12-03T05:16:02,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bbfbacaec36842c38c8f36a666fd3a05, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733202959087 2024-12-03T05:16:02,151 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9922de70d2be4db29e1523acce6495ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742114_1290 (size=12493) 2024-12-03T05:16:02,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T05:16:02,178 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:02,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,179 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:16:02,179 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/b57c957f959f4052a6514f8386e49e8b is 50, key is test_row_0/C:col10/1733202960348/Put/seqid=0 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/06e3c2ca7d0641738c8461a877ad08b2 is 50, key is test_row_0/A:col10/1733202961594/Put/seqid=0 2024-12-03T05:16:02,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742115_1291 (size=12493) 2024-12-03T05:16:02,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742116_1292 (size=12151) 2024-12-03T05:16:02,247 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/06e3c2ca7d0641738c8461a877ad08b2 2024-12-03T05:16:02,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:02,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/6d98a486cce64b15832435f7e1d061e8 is 50, key is test_row_0/B:col10/1733202961594/Put/seqid=0 2024-12-03T05:16:02,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742117_1293 (size=12151) 2024-12-03T05:16:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/6d98a486cce64b15832435f7e1d061e8 2024-12-03T05:16:02,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8855fa67d3b341bc95277fd310582061 is 50, key is test_row_0/C:col10/1733202961594/Put/seqid=0 2024-12-03T05:16:02,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742118_1294 (size=12151) 2024-12-03T05:16:02,418 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8855fa67d3b341bc95277fd310582061 2024-12-03T05:16:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/06e3c2ca7d0641738c8461a877ad08b2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2 2024-12-03T05:16:02,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T05:16:02,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/6d98a486cce64b15832435f7e1d061e8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8 2024-12-03T05:16:02,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T05:16:02,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8855fa67d3b341bc95277fd310582061 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061 2024-12-03T05:16:02,455 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T05:16:02,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=100.63 KB/103050 for 337aee72174b5cc395d13e96dd50849b in 279ms, sequenceid=166, compaction requested=false 2024-12-03T05:16:02,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:02,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T05:16:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T05:16:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:02,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:02,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T05:16:02,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-12-03T05:16:02,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/254cd093b94840d6bbe45c71744ebe00 is 50, key is test_row_0/A:col10/1733202962413/Put/seqid=0 2024-12-03T05:16:02,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.1020 sec 2024-12-03T05:16:02,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742119_1295 (size=14541) 2024-12-03T05:16:02,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203022510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,514 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203022514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203022514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203022515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T05:16:02,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:02,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T05:16:02,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203022520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:02,524 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:02,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:02,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:02,584 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/9125bc7e11f34bc492acd7a23b0d093a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9125bc7e11f34bc492acd7a23b0d093a 2024-12-03T05:16:02,590 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into 9125bc7e11f34bc492acd7a23b0d093a(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:02,590 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:02,590 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=13, startTime=1733202962108; duration=0sec 2024-12-03T05:16:02,591 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:02,591 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203022621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203022621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:02,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203022630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,638 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/b57c957f959f4052a6514f8386e49e8b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/b57c957f959f4052a6514f8386e49e8b 2024-12-03T05:16:02,643 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into b57c957f959f4052a6514f8386e49e8b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:02,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:02,643 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=13, startTime=1733202962109; duration=0sec 2024-12-03T05:16:02,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:02,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:02,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203022830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203022831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203022841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:02,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/254cd093b94840d6bbe45c71744ebe00 2024-12-03T05:16:02,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/98f097e9b2274bedada3b6e67421c2a6 is 50, key is test_row_0/B:col10/1733202962413/Put/seqid=0 2024-12-03T05:16:02,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742120_1296 (size=12151) 2024-12-03T05:16:02,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:02,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203023145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203023144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203023150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/98f097e9b2274bedada3b6e67421c2a6 2024-12-03T05:16:03,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ae204284294aa8bba28af7fcaae0d1 is 50, key is test_row_0/C:col10/1733202962413/Put/seqid=0 2024-12-03T05:16:03,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742121_1297 (size=12151) 2024-12-03T05:16:03,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ae204284294aa8bba28af7fcaae0d1 2024-12-03T05:16:03,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:03,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/254cd093b94840d6bbe45c71744ebe00 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00 2024-12-03T05:16:03,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00, entries=200, sequenceid=186, filesize=14.2 K 2024-12-03T05:16:03,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/98f097e9b2274bedada3b6e67421c2a6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6 2024-12-03T05:16:03,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6, entries=150, sequenceid=186, filesize=11.9 K 2024-12-03T05:16:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ae204284294aa8bba28af7fcaae0d1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1 2024-12-03T05:16:03,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1, entries=150, sequenceid=186, filesize=11.9 K 2024-12-03T05:16:03,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 337aee72174b5cc395d13e96dd50849b in 1047ms, sequenceid=186, compaction requested=true 2024-12-03T05:16:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:03,519 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:03,519 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:03,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:03,524 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:03,524 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:03,524 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,524 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fe748fe920af42309a09199afff5576b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=35.9 K 2024-12-03T05:16:03,528 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:03,528 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fe748fe920af42309a09199afff5576b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:03,529 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:03,529 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,529 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9125bc7e11f34bc492acd7a23b0d093a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=38.3 K 2024-12-03T05:16:03,529 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9125bc7e11f34bc492acd7a23b0d093a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:03,530 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d98a486cce64b15832435f7e1d061e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733202961577 2024-12-03T05:16:03,532 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e3c2ca7d0641738c8461a877ad08b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733202961577 2024-12-03T05:16:03,532 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f097e9b2274bedada3b6e67421c2a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733202962413 2024-12-03T05:16:03,536 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254cd093b94840d6bbe45c71744ebe00, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733202962413 2024-12-03T05:16:03,567 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:03,568 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b32dd9fb6f4f47a29cf27e7380535b5f is 50, key is test_row_0/A:col10/1733202962413/Put/seqid=0 2024-12-03T05:16:03,570 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:03,571 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e8b2ce27fb25420aa4ab08a9cb429b28 is 50, key is test_row_0/B:col10/1733202962413/Put/seqid=0 2024-12-03T05:16:03,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742122_1298 (size=12595) 2024-12-03T05:16:03,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742123_1299 (size=12595) 2024-12-03T05:16:03,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T05:16:03,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,644 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2b9d6fc4e9e94d6ba3a66587cd182f85 is 50, key is test_row_0/A:col10/1733202962511/Put/seqid=0 2024-12-03T05:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:03,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742124_1300 (size=9757) 2024-12-03T05:16:03,676 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2b9d6fc4e9e94d6ba3a66587cd182f85 2024-12-03T05:16:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/9029cc37329848ffa8773777821c2a8e is 50, key is test_row_0/B:col10/1733202962511/Put/seqid=0 2024-12-03T05:16:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742125_1301 (size=9757) 2024-12-03T05:16:03,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203023706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203023707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,712 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/9029cc37329848ffa8773777821c2a8e 2024-12-03T05:16:03,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203023709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ebeea0968f4c1b8719ff70c17ef6a5 is 50, key is test_row_0/C:col10/1733202962511/Put/seqid=0 2024-12-03T05:16:03,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742126_1302 (size=9757) 2024-12-03T05:16:03,771 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ebeea0968f4c1b8719ff70c17ef6a5 2024-12-03T05:16:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/2b9d6fc4e9e94d6ba3a66587cd182f85 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85 2024-12-03T05:16:03,792 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:16:03,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/9029cc37329848ffa8773777821c2a8e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e 2024-12-03T05:16:03,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:16:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/64ebeea0968f4c1b8719ff70c17ef6a5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5 2024-12-03T05:16:03,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:16:03,819 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 174ms, sequenceid=201, compaction requested=true 2024-12-03T05:16:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T05:16:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T05:16:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:03,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:03,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/8a3e2abf9c60463e80a104086995ded3 is 50, key is test_row_0/A:col10/1733202963703/Put/seqid=0 2024-12-03T05:16:03,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T05:16:03,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-12-03T05:16:03,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3300 sec 2024-12-03T05:16:03,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742127_1303 (size=14541) 2024-12-03T05:16:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203023870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203023885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203023888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203023990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203023990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203024001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,059 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b32dd9fb6f4f47a29cf27e7380535b5f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b32dd9fb6f4f47a29cf27e7380535b5f 2024-12-03T05:16:04,071 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e8b2ce27fb25420aa4ab08a9cb429b28 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8b2ce27fb25420aa4ab08a9cb429b28 2024-12-03T05:16:04,091 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into b32dd9fb6f4f47a29cf27e7380535b5f(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:04,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:04,091 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=13, startTime=1733202963519; duration=0sec 2024-12-03T05:16:04,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:04,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:04,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:04,099 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into e8b2ce27fb25420aa4ab08a9cb429b28(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:04,099 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:04,099 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:04,099 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=13, startTime=1733202963519; duration=0sec 2024-12-03T05:16:04,099 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:04,099 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:04,099 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:04,099 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:04,099 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/b57c957f959f4052a6514f8386e49e8b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=45.5 K 2024-12-03T05:16:04,100 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57c957f959f4052a6514f8386e49e8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733202960322 2024-12-03T05:16:04,100 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8855fa67d3b341bc95277fd310582061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733202961577 2024-12-03T05:16:04,101 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ae204284294aa8bba28af7fcaae0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733202962413 2024-12-03T05:16:04,101 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ebeea0968f4c1b8719ff70c17ef6a5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733202962511 2024-12-03T05:16:04,113 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:04,114 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/4fe1e0707caf4834bb6acc8649661ded is 50, key is test_row_0/C:col10/1733202962511/Put/seqid=0 2024-12-03T05:16:04,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742128_1304 (size=12629) 2024-12-03T05:16:04,185 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/4fe1e0707caf4834bb6acc8649661ded as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe1e0707caf4834bb6acc8649661ded 2024-12-03T05:16:04,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203024205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203024205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203024213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,232 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into 4fe1e0707caf4834bb6acc8649661ded(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:04,233 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:04,233 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=12, startTime=1733202963519; duration=0sec 2024-12-03T05:16:04,233 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:04,233 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:04,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/8a3e2abf9c60463e80a104086995ded3 2024-12-03T05:16:04,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/10bca63a3c7f49198a883edf045ddc40 is 50, key is test_row_0/B:col10/1733202963703/Put/seqid=0 2024-12-03T05:16:04,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742129_1305 (size=12151) 2024-12-03T05:16:04,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/10bca63a3c7f49198a883edf045ddc40 2024-12-03T05:16:04,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/c04d66286b0a41a5be9d7edf87275aa2 is 50, key is test_row_0/C:col10/1733202963703/Put/seqid=0 2024-12-03T05:16:04,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742130_1306 (size=12151) 2024-12-03T05:16:04,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203024521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203024521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203024530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T05:16:04,687 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:04,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T05:16:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:16:04,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:04,704 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:04,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:16:04,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T05:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:04,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:04,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/c04d66286b0a41a5be9d7edf87275aa2 2024-12-03T05:16:04,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/8a3e2abf9c60463e80a104086995ded3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3 2024-12-03T05:16:04,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3, entries=200, sequenceid=221, filesize=14.2 K 2024-12-03T05:16:04,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/10bca63a3c7f49198a883edf045ddc40 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40 2024-12-03T05:16:04,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T05:16:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/c04d66286b0a41a5be9d7edf87275aa2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2 2024-12-03T05:16:04,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T05:16:04,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 337aee72174b5cc395d13e96dd50849b in 1106ms, sequenceid=221, compaction requested=true 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:04,928 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:04,928 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:04,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36893 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:04,930 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:04,930 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:04,930 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8b2ce27fb25420aa4ab08a9cb429b28, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=33.7 K 2024-12-03T05:16:04,930 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b32dd9fb6f4f47a29cf27e7380535b5f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=36.0 K 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b2ce27fb25420aa4ab08a9cb429b28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733202962413 2024-12-03T05:16:04,930 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b32dd9fb6f4f47a29cf27e7380535b5f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733202962413 2024-12-03T05:16:04,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9029cc37329848ffa8773777821c2a8e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733202962511 2024-12-03T05:16:04,931 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9d6fc4e9e94d6ba3a66587cd182f85, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733202962511 2024-12-03T05:16:04,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 10bca63a3c7f49198a883edf045ddc40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733202963690 2024-12-03T05:16:04,931 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3e2abf9c60463e80a104086995ded3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733202963690 2024-12-03T05:16:04,939 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:04,939 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:04,940 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/cd6a4504b63142eca718a4e1b35631e8 is 50, key is test_row_0/A:col10/1733202963703/Put/seqid=0 2024-12-03T05:16:04,941 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/6ee4bad4417e4ecc82469133a5d8f340 is 50, key is test_row_0/B:col10/1733202963703/Put/seqid=0 2024-12-03T05:16:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742132_1308 (size=12697) 2024-12-03T05:16:04,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742131_1307 (size=12697) 2024-12-03T05:16:04,959 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/cd6a4504b63142eca718a4e1b35631e8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/cd6a4504b63142eca718a4e1b35631e8 2024-12-03T05:16:04,972 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into cd6a4504b63142eca718a4e1b35631e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:04,972 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:04,972 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=13, startTime=1733202964928; duration=0sec 2024-12-03T05:16:04,972 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:04,972 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:04,972 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:16:04,973 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:16:04,973 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:16:04,973 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. because compaction request was cancelled 2024-12-03T05:16:04,973 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:16:05,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T05:16:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:05,023 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:05,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:05,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b9e9cb75d3da4caab2c76bd19849dcf5 is 50, key is test_row_0/A:col10/1733202963877/Put/seqid=0 2024-12-03T05:16:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742133_1309 (size=12151) 2024-12-03T05:16:05,073 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b9e9cb75d3da4caab2c76bd19849dcf5 2024-12-03T05:16:05,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203025074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203025075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203025075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/99fab39b74854a12abc5c9e761ec30aa is 50, key is test_row_0/B:col10/1733202963877/Put/seqid=0 2024-12-03T05:16:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742134_1310 (size=12151) 2024-12-03T05:16:05,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203025191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203025191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203025191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:16:05,354 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/6ee4bad4417e4ecc82469133a5d8f340 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6ee4bad4417e4ecc82469133a5d8f340 2024-12-03T05:16:05,361 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into 6ee4bad4417e4ecc82469133a5d8f340(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:05,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:05,362 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=13, startTime=1733202964928; duration=0sec 2024-12-03T05:16:05,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:05,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:05,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203025411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203025411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203025411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:05,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/99fab39b74854a12abc5c9e761ec30aa 2024-12-03T05:16:05,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8f0326f6d21b47a597bf4eb70e3ea48e is 50, key is test_row_0/C:col10/1733202963877/Put/seqid=0 2024-12-03T05:16:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742135_1311 (size=12151) 2024-12-03T05:16:05,533 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8f0326f6d21b47a597bf4eb70e3ea48e 2024-12-03T05:16:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b9e9cb75d3da4caab2c76bd19849dcf5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5 2024-12-03T05:16:05,553 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T05:16:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/99fab39b74854a12abc5c9e761ec30aa as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa 2024-12-03T05:16:05,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T05:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8f0326f6d21b47a597bf4eb70e3ea48e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e 2024-12-03T05:16:05,570 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T05:16:05,571 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 548ms, sequenceid=238, compaction requested=true 2024-12-03T05:16:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T05:16:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T05:16:05,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T05:16:05,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-12-03T05:16:05,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 895 msec 2024-12-03T05:16:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:05,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202965066, max timestamp=1733202965723Num uniques 859; 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202965066, max timestamp=1733202965723Num uniques 859; 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202965066, max timestamp=1733202965723Num uniques 859; 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:05,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:05,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:05,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0ec4d1ec6bfa42f4b22a47079bba84e0 is 50, key is test_row_0/A:col10/1733202965723/Put/seqid=0 2024-12-03T05:16:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742136_1312 (size=17081) 2024-12-03T05:16:05,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0ec4d1ec6bfa42f4b22a47079bba84e0 2024-12-03T05:16:05,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/b60adb7cd270441291b85e49416c339a is 50, key is test_row_0/B:col10/1733202965723/Put/seqid=0 2024-12-03T05:16:05,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733202965742, max timestamp=1733202965764Num uniques 859; 2024-12-03T05:16:05,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:05,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202965755, max timestamp=1733202965768Num uniques 859; 2024-12-03T05:16:05,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:05,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202965755, max timestamp=1733202965768Num uniques 859; 2024-12-03T05:16:05,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:05,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742137_1313 (size=12251) 2024-12-03T05:16:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T05:16:05,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:05,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-03T05:16:05,837 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:05,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:05,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:05,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202965810, max timestamp=1733202965829Num uniques 1068; 2024-12-03T05:16:05,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-03T05:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202965810, max timestamp=1733202965829Num uniques 1018; 2024-12-03T05:16:05,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-03T05:16:05,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202965810, max timestamp=1733202965829Num uniques 1018; 2024-12-03T05:16:05,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-03T05:16:05,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733202965867, max timestamp=1733202965888Num uniques 1009; 2024-12-03T05:16:05,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-03T05:16:05,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202965867, max timestamp=1733202965888Num uniques 1009; 2024-12-03T05:16:05,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-03T05:16:05,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733202965867, max timestamp=1733202965888Num uniques 1009; 2024-12-03T05:16:05,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-03T05:16:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202965918, max timestamp=1733202965941Num uniques 1036; 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202965918, max timestamp=1733202965941Num uniques 1036; 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733202965918, max timestamp=1733202965941Num uniques 1009; 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:16:05,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-12-03T05:16:05,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733202965963, max timestamp=1733202965976Num uniques 1045; 2024-12-03T05:16:05,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733202965963, max timestamp=1733202965976Num uniques 1045; 2024-12-03T05:16:05,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:16:05,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:16:05,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733202965963, max timestamp=1733202965976Num uniques 1045; 2024-12-03T05:16:05,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:16:05,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:05,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202966006, max timestamp=1733202966015Num uniques 1054; 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202966006, max timestamp=1733202966015Num uniques 1054; 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733202966006, max timestamp=1733202966015Num uniques 1054; 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-03T05:16:06,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-03T05:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733202966029, max timestamp=1733202966046Num uniques 1013; 2024-12-03T05:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-12-03T05:16:06,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733202966039, max timestamp=1733202966046Num uniques 1013; 2024-12-03T05:16:06,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733202966039, max timestamp=1733202966046Num uniques 1013; 2024-12-03T05:16:06,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T05:16:06,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T05:16:06,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733202966063, max timestamp=1733202966089Num uniques 1022; 2024-12-03T05:16:06,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733202966063, max timestamp=1733202966089Num uniques 1122; 2024-12-03T05:16:06,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-03T05:16:06,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-03T05:16:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733202966063, max timestamp=1733202966089Num uniques 1022; 2024-12-03T05:16:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-03T05:16:06,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733202966108, max timestamp=1733202966144Num uniques 1081; 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733202966108, max timestamp=1733202966144Num uniques 1081; 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733202966108, max timestamp=1733202966144Num uniques 1081; 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-03T05:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-03T05:16:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:06,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/b60adb7cd270441291b85e49416c339a 2024-12-03T05:16:06,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/725be1c0c5a244c5a54ce8de1a7242d7 is 50, key is test_row_0/C:col10/1733202965723/Put/seqid=0 2024-12-03T05:16:06,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733202966181, max timestamp=1733202966190Num uniques 1040; 2024-12-03T05:16:06,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-03T05:16:06,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733202966181, max timestamp=1733202966190Num uniques 1040; 2024-12-03T05:16:06,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-03T05:16:06,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733202966181, max timestamp=1733202966190Num uniques 1040; 2024-12-03T05:16:06,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-03T05:16:06,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742138_1314 (size=12251) 2024-12-03T05:16:06,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733202966228, max timestamp=1733202966239Num uniques 1049; 2024-12-03T05:16:06,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-03T05:16:06,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9107, totalHeapSize=20936, min timestamp=1733202966228, max timestamp=1733202966239Num uniques 1049; 2024-12-03T05:16:06,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48037, new segment data size=9107, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-03T05:16:06,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733202966228, max timestamp=1733202966239Num uniques 1049; 2024-12-03T05:16:06,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733202966273, max timestamp=1733202966293Num uniques 1058; 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733202966273, max timestamp=1733202966293Num uniques 1058; 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733202966273, max timestamp=1733202966293Num uniques 1058; 2024-12-03T05:16:06,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-03T05:16:06,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733202966351, max timestamp=1733202966369Num uniques 1117; 2024-12-03T05:16:06,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-03T05:16:06,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733202966351, max timestamp=1733202966369Num uniques 1117; 2024-12-03T05:16:06,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-03T05:16:06,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202966352, max timestamp=1733202966369Num uniques 1117; 2024-12-03T05:16:06,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=6870, suffix heap size=144136,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=150 2024-12-03T05:16:06,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733202966399, max timestamp=1733202966417Num uniques 1076; 2024-12-03T05:16:06,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-03T05:16:06,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733202966399, max timestamp=1733202966417Num uniques 1076; 2024-12-03T05:16:06,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-03T05:16:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733202966399, max timestamp=1733202966417Num uniques 1009; 2024-12-03T05:16:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8056, suffix heap size=132904,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=176 2024-12-03T05:16:06,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:06,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733202966447, max timestamp=1733202966462Num uniques 1085; 2024-12-03T05:16:06,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-12-03T05:16:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733202966447, max timestamp=1733202966462Num uniques 1085; 2024-12-03T05:16:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-12-03T05:16:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733202966447, max timestamp=1733202966462Num uniques 1035; 2024-12-03T05:16:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-03T05:16:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733202966507, max timestamp=1733202966515Num uniques 1094; 2024-12-03T05:16:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-12-03T05:16:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733202966507, max timestamp=1733202966515Num uniques 1094; 2024-12-03T05:16:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-12-03T05:16:06,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733202966507, max timestamp=1733202966515Num uniques 1094; 2024-12-03T05:16:06,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=6870, suffix heap size=141744,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=150 2024-12-03T05:16:06,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733202966547, max timestamp=1733202966560Num uniques 1053; 2024-12-03T05:16:06,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-03T05:16:06,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733202966547, max timestamp=1733202966560Num uniques 1009; 2024-12-03T05:16:06,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=7005, suffix heap size=132904,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=153 2024-12-03T05:16:06,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733202966547, max timestamp=1733202966560Num uniques 1053; 2024-12-03T05:16:06,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-03T05:16:06,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203026589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203026589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9692, totalHeapSize=22288, min timestamp=1733202966576, max timestamp=1733202966593Num uniques 1062; 2024-12-03T05:16:06,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48622, new segment data size=9692, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-03T05:16:06,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9692, totalHeapSize=22288, min timestamp=1733202966576, max timestamp=1733202966593Num uniques 1062; 2024-12-03T05:16:06,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48622, new segment data size=9692, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-03T05:16:06,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9700, totalHeapSize=22288, min timestamp=1733202966576, max timestamp=1733202966593Num uniques 1012; 2024-12-03T05:16:06,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46340, new segment data size=9700, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-12-03T05:16:06,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/725be1c0c5a244c5a54ce8de1a7242d7 2024-12-03T05:16:06,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0ec4d1ec6bfa42f4b22a47079bba84e0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0 2024-12-03T05:16:06,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0, entries=250, sequenceid=261, filesize=16.7 K 2024-12-03T05:16:06,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/b60adb7cd270441291b85e49416c339a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a 2024-12-03T05:16:06,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a, entries=150, sequenceid=261, filesize=12.0 K 2024-12-03T05:16:06,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/725be1c0c5a244c5a54ce8de1a7242d7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7 2024-12-03T05:16:06,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7, entries=150, sequenceid=261, filesize=12.0 K 2024-12-03T05:16:06,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 915ms, sequenceid=261, compaction requested=true 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:06,639 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:06,639 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:06,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:06,640 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:06,640 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:06,640 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,640 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/cd6a4504b63142eca718a4e1b35631e8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=40.9 K 2024-12-03T05:16:06,640 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:06,640 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:06,641 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,641 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6ee4bad4417e4ecc82469133a5d8f340, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=36.2 K 2024-12-03T05:16:06,641 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6a4504b63142eca718a4e1b35631e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733202963690 2024-12-03T05:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:06,642 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee4bad4417e4ecc82469133a5d8f340, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733202963690 2024-12-03T05:16:06,642 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e9cb75d3da4caab2c76bd19849dcf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733202963847 2024-12-03T05:16:06,642 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec4d1ec6bfa42f4b22a47079bba84e0, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733202965066 2024-12-03T05:16:06,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fab39b74854a12abc5c9e761ec30aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733202963847 2024-12-03T05:16:06,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=120.76 KB heapSize=340.17 KB 2024-12-03T05:16:06,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting b60adb7cd270441291b85e49416c339a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733202965073 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:06,658 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:06,659 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e8a1beaa19a04631989cf983202c42da is 50, key is test_row_0/B:col10/1733202965723/Put/seqid=0 2024-12-03T05:16:06,663 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:06,663 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/db451e968b244b0c926176380748962b is 50, key is test_row_0/A:col10/1733202965723/Put/seqid=0 2024-12-03T05:16:06,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e25e9e275ac4cb2b3d3738f0b08a311 is 50, key is test_row_0/A:col10/1733202966593/Put/seqid=0 2024-12-03T05:16:06,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203026665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203026670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203026670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203026700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203026704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742139_1315 (size=12899) 2024-12-03T05:16:06,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742141_1317 (size=12301) 2024-12-03T05:16:06,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e25e9e275ac4cb2b3d3738f0b08a311 2024-12-03T05:16:06,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742140_1316 (size=12899) 2024-12-03T05:16:06,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/ee2e31a27d1f4b32b023e1a89537f41a is 50, key is test_row_0/B:col10/1733202966593/Put/seqid=0 2024-12-03T05:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203026780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203026781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203026781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742142_1318 (size=12301) 2024-12-03T05:16:06,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/ee2e31a27d1f4b32b023e1a89537f41a 2024-12-03T05:16:06,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7c801bb421a344d986ec4a44516b9efd is 50, key is test_row_0/C:col10/1733202966593/Put/seqid=0 2024-12-03T05:16:06,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742143_1319 (size=12301) 2024-12-03T05:16:06,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7c801bb421a344d986ec4a44516b9efd 2024-12-03T05:16:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3e25e9e275ac4cb2b3d3738f0b08a311 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311 2024-12-03T05:16:06,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311, entries=150, sequenceid=587, filesize=12.0 K 2024-12-03T05:16:06,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/ee2e31a27d1f4b32b023e1a89537f41a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a 2024-12-03T05:16:06,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203026910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a, entries=150, sequenceid=587, filesize=12.0 K 2024-12-03T05:16:06,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7c801bb421a344d986ec4a44516b9efd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd 2024-12-03T05:16:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203026922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:06,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd, entries=150, sequenceid=587, filesize=12.0 K 2024-12-03T05:16:06,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.13 KB/348288, currentSize=60.38 KB/61830 for 337aee72174b5cc395d13e96dd50849b in 284ms, sequenceid=587, compaction requested=true 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:06,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:16:06,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T05:16:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:06,934 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:06,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7d1624efba5e476384ddd9b7e8af371b is 50, key is test_row_0/A:col10/1733202966666/Put/seqid=0 2024-12-03T05:16:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:06,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742144_1320 (size=12301) 2024-12-03T05:16:06,990 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7d1624efba5e476384ddd9b7e8af371b 2024-12-03T05:16:06,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:07,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c54f30b73e8543fb967d8f5c1d2d54c4 is 50, key is test_row_0/B:col10/1733202966666/Put/seqid=0 2024-12-03T05:16:07,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742145_1321 (size=12301) 2024-12-03T05:16:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203027046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203027046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203027049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,136 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e8a1beaa19a04631989cf983202c42da as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8a1beaa19a04631989cf983202c42da 2024-12-03T05:16:07,149 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into e8a1beaa19a04631989cf983202c42da(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:07,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:07,149 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=13, startTime=1733202966639; duration=0sec 2024-12-03T05:16:07,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T05:16:07,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:07,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:16:07,153 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61483 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:16:07,153 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:07,153 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:07,153 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe1e0707caf4834bb6acc8649661ded, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=60.0 K 2024-12-03T05:16:07,154 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe1e0707caf4834bb6acc8649661ded, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733202962464 2024-12-03T05:16:07,159 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c04d66286b0a41a5be9d7edf87275aa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733202963690 2024-12-03T05:16:07,182 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0326f6d21b47a597bf4eb70e3ea48e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733202963847 2024-12-03T05:16:07,182 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 725be1c0c5a244c5a54ce8de1a7242d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733202965073 2024-12-03T05:16:07,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203027178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203027178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203027179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,187 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/db451e968b244b0c926176380748962b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/db451e968b244b0c926176380748962b 2024-12-03T05:16:07,193 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c801bb421a344d986ec4a44516b9efd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733202966593 2024-12-03T05:16:07,196 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into db451e968b244b0c926176380748962b(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:07,196 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:07,197 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=13, startTime=1733202966639; duration=0sec 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. because compaction request was cancelled 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:07,197 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:16:07,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:16:07,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:16:07,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. because compaction request was cancelled 2024-12-03T05:16:07,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:07,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:16:07,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:16:07,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:16:07,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. because compaction request was cancelled 2024-12-03T05:16:07,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:07,211 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:07,212 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/278726b5864e4d0a99ac7bbf1b1be720 is 50, key is test_row_0/C:col10/1733202966593/Put/seqid=0 2024-12-03T05:16:07,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203027221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203027230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742146_1322 (size=12949) 2024-12-03T05:16:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203027391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203027390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203027391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,409 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c54f30b73e8543fb967d8f5c1d2d54c4 2024-12-03T05:16:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/39f2caa86ab942019785eb678dab642a is 50, key is test_row_0/C:col10/1733202966666/Put/seqid=0 2024-12-03T05:16:07,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742147_1323 (size=12301) 2024-12-03T05:16:07,643 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/278726b5864e4d0a99ac7bbf1b1be720 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/278726b5864e4d0a99ac7bbf1b1be720 2024-12-03T05:16:07,649 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into 278726b5864e4d0a99ac7bbf1b1be720(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:07,649 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:07,649 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=11, startTime=1733202966928; duration=0sec 2024-12-03T05:16:07,650 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:07,650 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203027700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203027700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203027710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203027740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203027748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:07,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/39f2caa86ab942019785eb678dab642a 2024-12-03T05:16:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7d1624efba5e476384ddd9b7e8af371b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b 2024-12-03T05:16:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b, entries=150, sequenceid=599, filesize=12.0 K 2024-12-03T05:16:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c54f30b73e8543fb967d8f5c1d2d54c4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4 2024-12-03T05:16:07,845 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4, entries=150, sequenceid=599, filesize=12.0 K 2024-12-03T05:16:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/39f2caa86ab942019785eb678dab642a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a 2024-12-03T05:16:07,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a, entries=150, sequenceid=599, filesize=12.0 K 2024-12-03T05:16:07,869 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 337aee72174b5cc395d13e96dd50849b in 935ms, sequenceid=599, compaction requested=true 2024-12-03T05:16:07,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:07,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:07,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-03T05:16:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-03T05:16:07,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T05:16:07,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-12-03T05:16:07,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0410 sec 2024-12-03T05:16:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T05:16:07,981 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:07,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-03T05:16:07,984 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:07,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:07,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:08,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T05:16:08,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:08,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:16:08,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:08,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7664c0ed5ad8443f9a4379a17411312b is 50, key is test_row_0/A:col10/1733202967045/Put/seqid=0 2024-12-03T05:16:08,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742148_1324 (size=12301) 2024-12-03T05:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203028243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203028244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203028245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203028351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203028351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203028352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203028560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203028560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203028560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,576 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7664c0ed5ad8443f9a4379a17411312b 2024-12-03T05:16:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/652f40a462064d158bcd69299ef9f981 is 50, key is test_row_0/B:col10/1733202967045/Put/seqid=0 2024-12-03T05:16:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742149_1325 (size=12301) 2024-12-03T05:16:08,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/652f40a462064d158bcd69299ef9f981 2024-12-03T05:16:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:08,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/1f00207519a549aca3e662c468147ae3 is 50, key is test_row_0/C:col10/1733202967045/Put/seqid=0 2024-12-03T05:16:08,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742150_1326 (size=12301) 2024-12-03T05:16:08,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203028761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203028763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,766 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:08,766 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203028871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203028871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203028880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/1f00207519a549aca3e662c468147ae3 2024-12-03T05:16:09,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/7664c0ed5ad8443f9a4379a17411312b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b 2024-12-03T05:16:09,056 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b, entries=150, sequenceid=622, filesize=12.0 K 2024-12-03T05:16:09,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/652f40a462064d158bcd69299ef9f981 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981 2024-12-03T05:16:09,067 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981, entries=150, sequenceid=622, filesize=12.0 K 2024-12-03T05:16:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/1f00207519a549aca3e662c468147ae3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3 2024-12-03T05:16:09,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3, entries=150, sequenceid=622, filesize=12.0 K 2024-12-03T05:16:09,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 337aee72174b5cc395d13e96dd50849b in 933ms, sequenceid=622, compaction requested=true 2024-12-03T05:16:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-03T05:16:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-03T05:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:09,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T05:16:09,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1100 sec 2024-12-03T05:16:09,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1470 sec 2024-12-03T05:16:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:09,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:09,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/880c37fe2fe94c2198a79deac9399740 is 50, key is test_row_0/A:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:09,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742151_1327 (size=12301) 2024-12-03T05:16:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203029419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203029422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203029421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203029531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203029531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203029540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203029740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203029744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203029750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:09,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/880c37fe2fe94c2198a79deac9399740 2024-12-03T05:16:09,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2f02515a5d57474c823cfab185015608 is 50, key is test_row_0/B:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742152_1328 (size=12301) 2024-12-03T05:16:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203030050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203030050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203030061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T05:16:10,142 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:10,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-03T05:16:10,153 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:10,155 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:10,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:10,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2f02515a5d57474c823cfab185015608 2024-12-03T05:16:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:10,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b753ab7e05444b59b4ab0bfba59bd50 is 50, key is test_row_0/C:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:10,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742153_1329 (size=12301) 2024-12-03T05:16:10,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T05:16:10,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:10,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T05:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203030560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203030561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203030570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T05:16:10,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:10,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:10,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b753ab7e05444b59b4ab0bfba59bd50 2024-12-03T05:16:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/880c37fe2fe94c2198a79deac9399740 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740 2024-12-03T05:16:10,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740, entries=150, sequenceid=636, filesize=12.0 K 2024-12-03T05:16:10,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2f02515a5d57474c823cfab185015608 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608 2024-12-03T05:16:10,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608, entries=150, sequenceid=636, filesize=12.0 K 2024-12-03T05:16:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b753ab7e05444b59b4ab0bfba59bd50 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50 2024-12-03T05:16:10,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50, entries=150, sequenceid=636, filesize=12.0 K 2024-12-03T05:16:10,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 1339ms, sequenceid=636, compaction requested=true 2024-12-03T05:16:10,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:10,723 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:16:10,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:10,724 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62103 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:16:10,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:10,724 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:10,724 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:16:10,725 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:10,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:10,725 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/db451e968b244b0c926176380748962b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=60.6 K 2024-12-03T05:16:10,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:10,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:10,725 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting db451e968b244b0c926176380748962b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733202965073 2024-12-03T05:16:10,725 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e25e9e275ac4cb2b3d3738f0b08a311, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733202966593 2024-12-03T05:16:10,726 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d1624efba5e476384ddd9b7e8af371b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733202966659 2024-12-03T05:16:10,726 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7664c0ed5ad8443f9a4379a17411312b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733202967035 2024-12-03T05:16:10,726 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62103 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:16:10,727 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:10,727 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,727 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8a1beaa19a04631989cf983202c42da, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=60.6 K 2024-12-03T05:16:10,727 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880c37fe2fe94c2198a79deac9399740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:10,727 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a1beaa19a04631989cf983202c42da, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733202965073 2024-12-03T05:16:10,727 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ee2e31a27d1f4b32b023e1a89537f41a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733202966593 2024-12-03T05:16:10,728 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c54f30b73e8543fb967d8f5c1d2d54c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733202966659 2024-12-03T05:16:10,728 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 652f40a462064d158bcd69299ef9f981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733202967035 2024-12-03T05:16:10,729 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f02515a5d57474c823cfab185015608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:10,743 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:10,744 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b40f8b5d2d534d208c0c9223ca41408a is 50, key is test_row_0/A:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:10,748 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#280 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:10,748 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/d2ae168756954acaae56d157003710e5 is 50, key is test_row_0/B:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:10,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742155_1331 (size=13119) 2024-12-03T05:16:10,767 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/d2ae168756954acaae56d157003710e5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d2ae168756954acaae56d157003710e5 2024-12-03T05:16:10,774 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into d2ae168756954acaae56d157003710e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:10,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:10,774 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=11, startTime=1733202970724; duration=0sec 2024-12-03T05:16:10,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:10,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:10,774 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:10,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742154_1330 (size=13119) 2024-12-03T05:16:10,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:10,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:10,778 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,778 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/278726b5864e4d0a99ac7bbf1b1be720, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=48.7 K 2024-12-03T05:16:10,779 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 278726b5864e4d0a99ac7bbf1b1be720, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1733202966593 2024-12-03T05:16:10,780 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f2caa86ab942019785eb678dab642a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733202966659 2024-12-03T05:16:10,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-03T05:16:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:10,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:10,782 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f00207519a549aca3e662c468147ae3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733202967035 2024-12-03T05:16:10,783 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b753ab7e05444b59b4ab0bfba59bd50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:10,785 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/b40f8b5d2d534d208c0c9223ca41408a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b40f8b5d2d534d208c0c9223ca41408a 2024-12-03T05:16:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:10,795 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into b40f8b5d2d534d208c0c9223ca41408a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:10,796 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:10,796 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=11, startTime=1733202970722; duration=0sec 2024-12-03T05:16:10,796 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:10,796 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:10,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f18968586d8741ac811156053259ccc2 is 50, key is test_row_0/A:col10/1733202969419/Put/seqid=0 2024-12-03T05:16:10,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742156_1332 (size=12301) 2024-12-03T05:16:10,823 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:10,823 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/79cc67e6aa964ba98999e0d560b11049 is 50, key is test_row_0/C:col10/1733202968244/Put/seqid=0 2024-12-03T05:16:10,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203030833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203030833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742157_1333 (size=13085) 2024-12-03T05:16:10,890 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/79cc67e6aa964ba98999e0d560b11049 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/79cc67e6aa964ba98999e0d560b11049 2024-12-03T05:16:10,899 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into 79cc67e6aa964ba98999e0d560b11049(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:10,899 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:10,900 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=12, startTime=1733202970725; duration=0sec 2024-12-03T05:16:10,900 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:10,900 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203030940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203030956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203031150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203031170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,217 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f18968586d8741ac811156053259ccc2 2024-12-03T05:16:11,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/fdf7565437f641d4b5f88cd4c72eeafe is 50, key is test_row_0/B:col10/1733202969419/Put/seqid=0 2024-12-03T05:16:11,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742158_1334 (size=12301) 2024-12-03T05:16:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203031460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203031481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203031570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203031570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,573 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,574 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203031590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,592 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 337aee72174b5cc395d13e96dd50849b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,648 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/fdf7565437f641d4b5f88cd4c72eeafe 2024-12-03T05:16:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/36ac4d02a3f64851b3d0cfa216016ddc is 50, key is test_row_0/C:col10/1733202969419/Put/seqid=0 2024-12-03T05:16:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742159_1335 (size=12301) 2024-12-03T05:16:11,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203031971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:11,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203031990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:12,058 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/36ac4d02a3f64851b3d0cfa216016ddc 2024-12-03T05:16:12,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f18968586d8741ac811156053259ccc2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2 2024-12-03T05:16:12,064 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2, entries=150, sequenceid=656, filesize=12.0 K 2024-12-03T05:16:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/fdf7565437f641d4b5f88cd4c72eeafe as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe 2024-12-03T05:16:12,068 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe, entries=150, sequenceid=656, filesize=12.0 K 2024-12-03T05:16:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/36ac4d02a3f64851b3d0cfa216016ddc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc 2024-12-03T05:16:12,073 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc, entries=150, sequenceid=656, filesize=12.0 K 2024-12-03T05:16:12,073 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 337aee72174b5cc395d13e96dd50849b in 1293ms, sequenceid=656, compaction requested=false 2024-12-03T05:16:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-03T05:16:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-03T05:16:12,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T05:16:12,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-12-03T05:16:12,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.9340 sec 2024-12-03T05:16:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T05:16:12,301 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:12,301 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-03T05:16:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T05:16:12,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:12,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:12,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T05:16:12,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-03T05:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:12,464 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:16:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0c4a3f6bee274e508719ed6889064f93 is 50, key is test_row_0/A:col10/1733202970815/Put/seqid=0 2024-12-03T05:16:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742160_1336 (size=12301) 2024-12-03T05:16:12,516 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0c4a3f6bee274e508719ed6889064f93 2024-12-03T05:16:12,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e88f3cb278fc4ad6a56760fd875808b2 is 50, key is test_row_0/B:col10/1733202970815/Put/seqid=0 2024-12-03T05:16:12,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742161_1337 (size=12301) 2024-12-03T05:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T05:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T05:16:12,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e88f3cb278fc4ad6a56760fd875808b2 2024-12-03T05:16:12,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/ee9ee7195363439eb06ec8551a901ca1 is 50, key is test_row_0/C:col10/1733202970815/Put/seqid=0 2024-12-03T05:16:12,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742162_1338 (size=12301) 2024-12-03T05:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:12,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203033020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203033021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203033130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203033130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203033340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203033341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,373 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/ee9ee7195363439eb06ec8551a901ca1 2024-12-03T05:16:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/0c4a3f6bee274e508719ed6889064f93 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93 2024-12-03T05:16:13,381 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93, entries=150, sequenceid=671, filesize=12.0 K 2024-12-03T05:16:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/e88f3cb278fc4ad6a56760fd875808b2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2 2024-12-03T05:16:13,385 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2, entries=150, sequenceid=671, filesize=12.0 K 2024-12-03T05:16:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/ee9ee7195363439eb06ec8551a901ca1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1 2024-12-03T05:16:13,389 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1, entries=150, sequenceid=671, filesize=12.0 K 2024-12-03T05:16:13,390 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 337aee72174b5cc395d13e96dd50849b in 926ms, sequenceid=671, compaction requested=true 2024-12-03T05:16:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-03T05:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-03T05:16:13,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T05:16:13,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-12-03T05:16:13,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0960 sec 2024-12-03T05:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T05:16:13,441 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:13,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-03T05:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:13,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:13,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:13,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:13,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:13,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f794f5d4cff24cf182b143633273a51f is 50, key is test_row_0/A:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:13,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742163_1339 (size=17181) 2024-12-03T05:16:13,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203033621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203033622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203033623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203033651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203033656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203033731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203033731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203033732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:13,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203033940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203033940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203033941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:13,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f794f5d4cff24cf182b143633273a51f 2024-12-03T05:16:14,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c30d01a0fe1748e2a82edffc63b9f213 is 50, key is test_row_0/B:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:14,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742164_1340 (size=12301) 2024-12-03T05:16:14,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47904 deadline: 1733203034161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47934 deadline: 1733203034171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203034251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203034251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203034252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c30d01a0fe1748e2a82edffc63b9f213 2024-12-03T05:16:14,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/6cf84142a8f24523a2f349a615025339 is 50, key is test_row_0/C:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:14,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742165_1341 (size=12301) 2024-12-03T05:16:14,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:14,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47924 deadline: 1733203034761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47952 deadline: 1733203034761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47892 deadline: 1733203034761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:14,773 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-12-03T05:16:14,773 INFO [Thread-1206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1206. 2024-12-03T05:16:14,773 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:14,773 DEBUG [Thread-1206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:14,773 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,773 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,773 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:14,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:14,774 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,776 INFO [Thread-1210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1210. 2024-12-03T05:16:14,776 DEBUG [Thread-1210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:14,776 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,776 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:14,777 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-12-03T05:16:14,777 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:14,777 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,778 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:14,778 INFO [Thread-1208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1208. 2024-12-03T05:16:14,778 DEBUG [Thread-1208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:14,778 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:14,779 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:14,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/6cf84142a8f24523a2f349a615025339 2024-12-03T05:16:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:14,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/f794f5d4cff24cf182b143633273a51f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f 2024-12-03T05:16:14,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f, entries=250, sequenceid=692, filesize=16.8 K 2024-12-03T05:16:14,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/c30d01a0fe1748e2a82edffc63b9f213 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213 2024-12-03T05:16:14,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213, entries=150, sequenceid=692, filesize=12.0 K 2024-12-03T05:16:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/6cf84142a8f24523a2f349a615025339 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339 2024-12-03T05:16:14,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339, entries=150, sequenceid=692, filesize=12.0 K 2024-12-03T05:16:14,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 337aee72174b5cc395d13e96dd50849b in 1278ms, sequenceid=692, compaction requested=true 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:14,862 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 337aee72174b5cc395d13e96dd50849b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:14,862 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:14,863 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:14,863 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:14,863 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/A is initiating minor compaction (all files) 2024-12-03T05:16:14,863 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/B is initiating minor compaction (all files) 2024-12-03T05:16:14,863 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/B in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,863 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/A in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,863 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b40f8b5d2d534d208c0c9223ca41408a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=53.6 K 2024-12-03T05:16:14,863 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d2ae168756954acaae56d157003710e5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=48.8 K 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40f8b5d2d534d208c0c9223ca41408a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ae168756954acaae56d157003710e5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf7565437f641d4b5f88cd4c72eeafe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1733202969411 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f18968586d8741ac811156053259ccc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1733202969411 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e88f3cb278fc4ad6a56760fd875808b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202970815 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4a3f6bee274e508719ed6889064f93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202970815 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c30d01a0fe1748e2a82edffc63b9f213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1733202973010 2024-12-03T05:16:14,864 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f794f5d4cff24cf182b143633273a51f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1733202973010 2024-12-03T05:16:14,878 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#B#compaction#291 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:14,878 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/0922d50c96594315b9e4c8dc959b8b2e is 50, key is test_row_0/B:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:14,905 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#A#compaction#292 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:14,906 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/9cef9270106d40098baa5daa69d6ace9 is 50, key is test_row_0/A:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:14,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742166_1342 (size=13255) 2024-12-03T05:16:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742167_1343 (size=13255) 2024-12-03T05:16:14,951 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/9cef9270106d40098baa5daa69d6ace9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9cef9270106d40098baa5daa69d6ace9 2024-12-03T05:16:14,957 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/A of 337aee72174b5cc395d13e96dd50849b into 9cef9270106d40098baa5daa69d6ace9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:14,958 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:14,958 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/A, priority=12, startTime=1733202974862; duration=0sec 2024-12-03T05:16:14,958 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:14,958 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:A 2024-12-03T05:16:14,958 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:14,959 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:14,959 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 337aee72174b5cc395d13e96dd50849b/C is initiating minor compaction (all files) 2024-12-03T05:16:14,959 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 337aee72174b5cc395d13e96dd50849b/C in TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,959 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/79cc67e6aa964ba98999e0d560b11049, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp, totalSize=48.8 K 2024-12-03T05:16:14,960 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cc67e6aa964ba98999e0d560b11049, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1733202968223 2024-12-03T05:16:14,961 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ac4d02a3f64851b3d0cfa216016ddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1733202969411 2024-12-03T05:16:14,962 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9ee7195363439eb06ec8551a901ca1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733202970815 2024-12-03T05:16:14,963 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf84142a8f24523a2f349a615025339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1733202973010 2024-12-03T05:16:14,985 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 337aee72174b5cc395d13e96dd50849b#C#compaction#293 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:14,986 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/fff4e2f347a145a1bf6d39dd85a0edae is 50, key is test_row_0/C:col10/1733202973010/Put/seqid=0 2024-12-03T05:16:14,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-03T05:16:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:14,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3eac234986214457ab2339afe4b33380 is 50, key is test_row_0/A:col10/1733202973619/Put/seqid=0 2024-12-03T05:16:15,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742168_1344 (size=13221) 2024-12-03T05:16:15,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742169_1345 (size=12301) 2024-12-03T05:16:15,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3eac234986214457ab2339afe4b33380 2024-12-03T05:16:15,042 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/fff4e2f347a145a1bf6d39dd85a0edae as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/fff4e2f347a145a1bf6d39dd85a0edae 2024-12-03T05:16:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/1c09219a01f94438a216ed6949e4f6df is 50, key is test_row_0/B:col10/1733202973619/Put/seqid=0 2024-12-03T05:16:15,049 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/C of 337aee72174b5cc395d13e96dd50849b into fff4e2f347a145a1bf6d39dd85a0edae(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:15,049 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:15,049 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/C, priority=12, startTime=1733202974862; duration=0sec 2024-12-03T05:16:15,049 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:15,049 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:C 2024-12-03T05:16:15,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742170_1346 (size=12301) 2024-12-03T05:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. as already flushing 2024-12-03T05:16:15,182 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-12-03T05:16:15,182 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:15,182 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,183 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:15,194 INFO [Thread-1199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1199. 2024-12-03T05:16:15,194 DEBUG [Thread-1199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:15,194 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,194 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:15,318 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/0922d50c96594315b9e4c8dc959b8b2e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/0922d50c96594315b9e4c8dc959b8b2e 2024-12-03T05:16:15,322 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 337aee72174b5cc395d13e96dd50849b/B of 337aee72174b5cc395d13e96dd50849b into 0922d50c96594315b9e4c8dc959b8b2e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:15,322 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:15,322 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., storeName=337aee72174b5cc395d13e96dd50849b/B, priority=12, startTime=1733202974862; duration=0sec 2024-12-03T05:16:15,323 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:15,323 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 337aee72174b5cc395d13e96dd50849b:B 2024-12-03T05:16:15,466 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/1c09219a01f94438a216ed6949e4f6df 2024-12-03T05:16:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7ce5c23502ff4e408a2c4dfb3a7479c4 is 50, key is test_row_0/C:col10/1733202973619/Put/seqid=0 2024-12-03T05:16:15,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742171_1347 (size=12301) 2024-12-03T05:16:15,482 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:15,772 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-12-03T05:16:15,772 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:15,772 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,772 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:15,782 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-12-03T05:16:15,782 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:15,782 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,782 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:15,782 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-12-03T05:16:15,782 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:15,782 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,782 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:15,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:15,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7ce5c23502ff4e408a2c4dfb3a7479c4 2024-12-03T05:16:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/3eac234986214457ab2339afe4b33380 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3eac234986214457ab2339afe4b33380 2024-12-03T05:16:15,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3eac234986214457ab2339afe4b33380, entries=150, sequenceid=704, filesize=12.0 K 2024-12-03T05:16:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/1c09219a01f94438a216ed6949e4f6df as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/1c09219a01f94438a216ed6949e4f6df 2024-12-03T05:16:15,889 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/1c09219a01f94438a216ed6949e4f6df, entries=150, sequenceid=704, filesize=12.0 K 2024-12-03T05:16:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/7ce5c23502ff4e408a2c4dfb3a7479c4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7ce5c23502ff4e408a2c4dfb3a7479c4 2024-12-03T05:16:15,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7ce5c23502ff4e408a2c4dfb3a7479c4, entries=150, sequenceid=704, filesize=12.0 K 2024-12-03T05:16:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for 337aee72174b5cc395d13e96dd50849b in 897ms, sequenceid=704, compaction requested=false 2024-12-03T05:16:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 337aee72174b5cc395d13e96dd50849b: 2024-12-03T05:16:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-03T05:16:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-03T05:16:15,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-03T05:16:15,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4520 sec 2024-12-03T05:16:15,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.4560 sec 2024-12-03T05:16:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T05:16:17,610 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:17,610 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:16:17,610 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 208 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2830 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2886 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2779 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2851 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2883 2024-12-03T05:16:17,611 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:16:17,611 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:16:17,611 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:16:17,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:17,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:17,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:17,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:16:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:17,616 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202977615"}]},"ts":"1733202977615"} 2024-12-03T05:16:17,617 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:16:17,618 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:16:17,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:16:17,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, UNASSIGN}] 2024-12-03T05:16:17,622 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, UNASSIGN 2024-12-03T05:16:17,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=337aee72174b5cc395d13e96dd50849b, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:17,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, UNASSIGN because future has completed 2024-12-03T05:16:17,624 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:16:17,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:17,777 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:17,777 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:16:17,777 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 337aee72174b5cc395d13e96dd50849b, disabling compactions & flushes 2024-12-03T05:16:17,777 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:17,777 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:17,777 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. after waiting 0 ms 2024-12-03T05:16:17,777 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:17,777 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 337aee72174b5cc395d13e96dd50849b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=A 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=B 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 337aee72174b5cc395d13e96dd50849b, store=C 2024-12-03T05:16:17,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:17,781 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/a0dcc095552f4c5682eb4ece05043ab7 is 50, key is test_row_1/A:col10/1733202975781/Put/seqid=0 2024-12-03T05:16:17,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742172_1348 (size=9857) 2024-12-03T05:16:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:18,184 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/a0dcc095552f4c5682eb4ece05043ab7 2024-12-03T05:16:18,191 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2b02352421664983a888386a0a41b635 is 50, key is test_row_1/B:col10/1733202975781/Put/seqid=0 2024-12-03T05:16:18,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742173_1349 (size=9857) 2024-12-03T05:16:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:18,602 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2b02352421664983a888386a0a41b635 2024-12-03T05:16:18,608 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b77bcb7a8a34ed49d9372aa03cd71d1 is 50, key is test_row_1/C:col10/1733202975781/Put/seqid=0 2024-12-03T05:16:18,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742174_1350 (size=9857) 2024-12-03T05:16:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:19,014 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b77bcb7a8a34ed49d9372aa03cd71d1 2024-12-03T05:16:19,018 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/A/a0dcc095552f4c5682eb4ece05043ab7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/a0dcc095552f4c5682eb4ece05043ab7 2024-12-03T05:16:19,022 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/a0dcc095552f4c5682eb4ece05043ab7, entries=100, sequenceid=714, filesize=9.6 K 2024-12-03T05:16:19,023 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/B/2b02352421664983a888386a0a41b635 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2b02352421664983a888386a0a41b635 2024-12-03T05:16:19,027 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2b02352421664983a888386a0a41b635, entries=100, sequenceid=714, filesize=9.6 K 2024-12-03T05:16:19,027 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/.tmp/C/8b77bcb7a8a34ed49d9372aa03cd71d1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b77bcb7a8a34ed49d9372aa03cd71d1 2024-12-03T05:16:19,032 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b77bcb7a8a34ed49d9372aa03cd71d1, entries=100, sequenceid=714, filesize=9.6 K 2024-12-03T05:16:19,033 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 337aee72174b5cc395d13e96dd50849b in 1256ms, sequenceid=714, compaction requested=true 2024-12-03T05:16:19,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/10fb3944faf84a5d930d3685ea3dfb0f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/84daa655f9bf4149aad238ac46a08ab3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9125bc7e11f34bc492acd7a23b0d093a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b32dd9fb6f4f47a29cf27e7380535b5f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/cd6a4504b63142eca718a4e1b35631e8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/db451e968b244b0c926176380748962b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b40f8b5d2d534d208c0c9223ca41408a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f] to archive 2024-12-03T05:16:19,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:19,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/81e50ec8c45c4b4c91bf0580c2ad8896 2024-12-03T05:16:19,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/637e7c0abedd44a98118950ce68fb7cb 2024-12-03T05:16:19,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/10fb3944faf84a5d930d3685ea3dfb0f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/10fb3944faf84a5d930d3685ea3dfb0f 2024-12-03T05:16:19,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2888bf411b76452e9f20d1c21cedf404 2024-12-03T05:16:19,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/baede85b05dd4369852e16f27fb7a3a3 2024-12-03T05:16:19,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/c59d8fc8d7dc4113bbd324709fc15d6b 2024-12-03T05:16:19,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/84daa655f9bf4149aad238ac46a08ab3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/84daa655f9bf4149aad238ac46a08ab3 2024-12-03T05:16:19,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e0877ffb8b0477380480054455666a0 2024-12-03T05:16:19,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0184c6a3dbfb41049f7e46399c94d3e7 2024-12-03T05:16:19,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9125bc7e11f34bc492acd7a23b0d093a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9125bc7e11f34bc492acd7a23b0d093a 2024-12-03T05:16:19,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7b467c937e21413d966a5d23e4928cbe 2024-12-03T05:16:19,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/06e3c2ca7d0641738c8461a877ad08b2 2024-12-03T05:16:19,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/254cd093b94840d6bbe45c71744ebe00 2024-12-03T05:16:19,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b32dd9fb6f4f47a29cf27e7380535b5f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b32dd9fb6f4f47a29cf27e7380535b5f 2024-12-03T05:16:19,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/2b9d6fc4e9e94d6ba3a66587cd182f85 2024-12-03T05:16:19,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/8a3e2abf9c60463e80a104086995ded3 2024-12-03T05:16:19,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/cd6a4504b63142eca718a4e1b35631e8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/cd6a4504b63142eca718a4e1b35631e8 2024-12-03T05:16:19,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b9e9cb75d3da4caab2c76bd19849dcf5 2024-12-03T05:16:19,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0ec4d1ec6bfa42f4b22a47079bba84e0 2024-12-03T05:16:19,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/db451e968b244b0c926176380748962b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/db451e968b244b0c926176380748962b 2024-12-03T05:16:19,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3e25e9e275ac4cb2b3d3738f0b08a311 2024-12-03T05:16:19,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7d1624efba5e476384ddd9b7e8af371b 2024-12-03T05:16:19,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/7664c0ed5ad8443f9a4379a17411312b 2024-12-03T05:16:19,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b40f8b5d2d534d208c0c9223ca41408a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/b40f8b5d2d534d208c0c9223ca41408a 2024-12-03T05:16:19,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/880c37fe2fe94c2198a79deac9399740 2024-12-03T05:16:19,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f18968586d8741ac811156053259ccc2 2024-12-03T05:16:19,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/0c4a3f6bee274e508719ed6889064f93 2024-12-03T05:16:19,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/f794f5d4cff24cf182b143633273a51f 2024-12-03T05:16:19,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e579fa2d4ae94cb2a3dc6784c56d70f8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/740fb04f6f7541e4bdc882aa0ede79a0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fe748fe920af42309a09199afff5576b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8b2ce27fb25420aa4ab08a9cb429b28, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6ee4bad4417e4ecc82469133a5d8f340, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8a1beaa19a04631989cf983202c42da, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d2ae168756954acaae56d157003710e5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213] to archive 2024-12-03T05:16:19,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:19,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d131561e9f93471986ee73c5dd41ccd8 2024-12-03T05:16:19,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2e6c0f52f45943d19bec7d7d4dc6f2c5 2024-12-03T05:16:19,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e579fa2d4ae94cb2a3dc6784c56d70f8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e579fa2d4ae94cb2a3dc6784c56d70f8 2024-12-03T05:16:19,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/58a0f82943fd49688fdcf60ac6570ec7 2024-12-03T05:16:19,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/04a533209d964bb8a484ef1ce3988f9b 2024-12-03T05:16:19,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2c9c166f1e764cce9ed2602ce0f6f6e2 2024-12-03T05:16:19,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/740fb04f6f7541e4bdc882aa0ede79a0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/740fb04f6f7541e4bdc882aa0ede79a0 2024-12-03T05:16:19,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/371c8a2c3ccd4e53834931662a21ee9d 2024-12-03T05:16:19,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e09e16c291834353ac8e9d44b499e8b7 2024-12-03T05:16:19,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fe748fe920af42309a09199afff5576b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fe748fe920af42309a09199afff5576b 2024-12-03T05:16:19,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/12862ba9803049d9a7898bb66d07b95d 2024-12-03T05:16:19,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6d98a486cce64b15832435f7e1d061e8 2024-12-03T05:16:19,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8b2ce27fb25420aa4ab08a9cb429b28 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8b2ce27fb25420aa4ab08a9cb429b28 2024-12-03T05:16:19,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/98f097e9b2274bedada3b6e67421c2a6 2024-12-03T05:16:19,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/9029cc37329848ffa8773777821c2a8e 2024-12-03T05:16:19,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6ee4bad4417e4ecc82469133a5d8f340 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/6ee4bad4417e4ecc82469133a5d8f340 2024-12-03T05:16:19,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/10bca63a3c7f49198a883edf045ddc40 2024-12-03T05:16:19,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/99fab39b74854a12abc5c9e761ec30aa 2024-12-03T05:16:19,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8a1beaa19a04631989cf983202c42da to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e8a1beaa19a04631989cf983202c42da 2024-12-03T05:16:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/b60adb7cd270441291b85e49416c339a 2024-12-03T05:16:19,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/ee2e31a27d1f4b32b023e1a89537f41a 2024-12-03T05:16:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c54f30b73e8543fb967d8f5c1d2d54c4 2024-12-03T05:16:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/652f40a462064d158bcd69299ef9f981 2024-12-03T05:16:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d2ae168756954acaae56d157003710e5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/d2ae168756954acaae56d157003710e5 2024-12-03T05:16:19,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2f02515a5d57474c823cfab185015608 2024-12-03T05:16:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/fdf7565437f641d4b5f88cd4c72eeafe 2024-12-03T05:16:19,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/e88f3cb278fc4ad6a56760fd875808b2 2024-12-03T05:16:19,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/c30d01a0fe1748e2a82edffc63b9f213 2024-12-03T05:16:19,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/d86f18bb866a411f877b2ce0c7a09296, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/531787a49e9d4003861c7a64e686e55c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/b57c957f959f4052a6514f8386e49e8b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe1e0707caf4834bb6acc8649661ded, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/278726b5864e4d0a99ac7bbf1b1be720, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/79cc67e6aa964ba98999e0d560b11049, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339] to archive 2024-12-03T05:16:19,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:19,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/777f53f734a44622a63fcce4fb01d2ac 2024-12-03T05:16:19,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/eb9eb1e0a26a4e549a025cae7f2afaf9 2024-12-03T05:16:19,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/d86f18bb866a411f877b2ce0c7a09296 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/d86f18bb866a411f877b2ce0c7a09296 2024-12-03T05:16:19,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9995cd59ea684c448cf516556711636a 2024-12-03T05:16:19,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/06cc3f785eb5407db8311f7f3112812f 2024-12-03T05:16:19,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe939ae0cb44b17b50ca15639c3ef6d 2024-12-03T05:16:19,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/531787a49e9d4003861c7a64e686e55c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/531787a49e9d4003861c7a64e686e55c 2024-12-03T05:16:19,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/23fd4a314e394707befa404851ce2718 2024-12-03T05:16:19,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/bbfbacaec36842c38c8f36a666fd3a05 2024-12-03T05:16:19,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/b57c957f959f4052a6514f8386e49e8b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/b57c957f959f4052a6514f8386e49e8b 2024-12-03T05:16:19,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/9922de70d2be4db29e1523acce6495ea 2024-12-03T05:16:19,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8855fa67d3b341bc95277fd310582061 2024-12-03T05:16:19,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ae204284294aa8bba28af7fcaae0d1 2024-12-03T05:16:19,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe1e0707caf4834bb6acc8649661ded to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/4fe1e0707caf4834bb6acc8649661ded 2024-12-03T05:16:19,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/64ebeea0968f4c1b8719ff70c17ef6a5 2024-12-03T05:16:19,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/c04d66286b0a41a5be9d7edf87275aa2 2024-12-03T05:16:19,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8f0326f6d21b47a597bf4eb70e3ea48e 2024-12-03T05:16:19,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/725be1c0c5a244c5a54ce8de1a7242d7 2024-12-03T05:16:19,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/278726b5864e4d0a99ac7bbf1b1be720 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/278726b5864e4d0a99ac7bbf1b1be720 2024-12-03T05:16:19,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7c801bb421a344d986ec4a44516b9efd 2024-12-03T05:16:19,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/39f2caa86ab942019785eb678dab642a 2024-12-03T05:16:19,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/1f00207519a549aca3e662c468147ae3 2024-12-03T05:16:19,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/79cc67e6aa964ba98999e0d560b11049 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/79cc67e6aa964ba98999e0d560b11049 2024-12-03T05:16:19,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b753ab7e05444b59b4ab0bfba59bd50 2024-12-03T05:16:19,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/36ac4d02a3f64851b3d0cfa216016ddc 2024-12-03T05:16:19,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/ee9ee7195363439eb06ec8551a901ca1 2024-12-03T05:16:19,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/6cf84142a8f24523a2f349a615025339 2024-12-03T05:16:19,150 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/recovered.edits/717.seqid, newMaxSeqId=717, maxSeqId=1 2024-12-03T05:16:19,151 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. 2024-12-03T05:16:19,151 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 337aee72174b5cc395d13e96dd50849b: Waiting for close lock at 1733202977777Running coprocessor pre-close hooks at 1733202977777Disabling compacts and flushes for region at 1733202977777Disabling writes for close at 1733202977777Obtaining lock to block concurrent updates at 1733202977777Preparing flush snapshotting stores in 337aee72174b5cc395d13e96dd50849b at 1733202977777Finished memstore snapshotting TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733202977778 (+1 ms)Flushing stores of TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b. at 1733202977778Flushing 337aee72174b5cc395d13e96dd50849b/A: creating writer at 1733202977778Flushing 337aee72174b5cc395d13e96dd50849b/A: appending metadata at 1733202977780 (+2 ms)Flushing 337aee72174b5cc395d13e96dd50849b/A: closing flushed file at 1733202977780Flushing 337aee72174b5cc395d13e96dd50849b/B: creating writer at 1733202978188 (+408 ms)Flushing 337aee72174b5cc395d13e96dd50849b/B: appending metadata at 1733202978190 (+2 ms)Flushing 337aee72174b5cc395d13e96dd50849b/B: closing flushed file at 1733202978190Flushing 337aee72174b5cc395d13e96dd50849b/C: creating writer at 1733202978606 (+416 ms)Flushing 337aee72174b5cc395d13e96dd50849b/C: appending metadata at 1733202978608 (+2 ms)Flushing 337aee72174b5cc395d13e96dd50849b/C: closing flushed file at 1733202978608Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@160d4f68: reopening flushed file at 1733202979018 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3ea09: reopening flushed file at 1733202979022 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@448a4dcc: reopening flushed file at 1733202979027 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 337aee72174b5cc395d13e96dd50849b in 1256ms, sequenceid=714, compaction requested=true at 1733202979033 (+6 ms)Writing region close event to WAL at 1733202979143 (+110 ms)Running coprocessor post-close hooks at 1733202979151 (+8 ms)Closed at 1733202979151 2024-12-03T05:16:19,153 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:19,154 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=337aee72174b5cc395d13e96dd50849b, regionState=CLOSED 2024-12-03T05:16:19,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:19,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-03T05:16:19,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 337aee72174b5cc395d13e96dd50849b, server=e3c6570475b6,37863,1733202887099 in 1.5330 sec 2024-12-03T05:16:19,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-03T05:16:19,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=337aee72174b5cc395d13e96dd50849b, UNASSIGN in 1.5390 sec 2024-12-03T05:16:19,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T05:16:19,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5480 sec 2024-12-03T05:16:19,176 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202979176"}]},"ts":"1733202979176"} 2024-12-03T05:16:19,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:16:19,185 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:16:19,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5740 sec 2024-12-03T05:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-03T05:16:19,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:19,761 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,763 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T05:16:19,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,766 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:19,768 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/recovered.edits] 2024-12-03T05:16:19,771 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3eac234986214457ab2339afe4b33380 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/3eac234986214457ab2339afe4b33380 2024-12-03T05:16:19,773 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9cef9270106d40098baa5daa69d6ace9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/9cef9270106d40098baa5daa69d6ace9 2024-12-03T05:16:19,774 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/a0dcc095552f4c5682eb4ece05043ab7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/A/a0dcc095552f4c5682eb4ece05043ab7 2024-12-03T05:16:19,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/0922d50c96594315b9e4c8dc959b8b2e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/0922d50c96594315b9e4c8dc959b8b2e 2024-12-03T05:16:19,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/1c09219a01f94438a216ed6949e4f6df to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/1c09219a01f94438a216ed6949e4f6df 2024-12-03T05:16:19,779 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2b02352421664983a888386a0a41b635 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/B/2b02352421664983a888386a0a41b635 2024-12-03T05:16:19,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7ce5c23502ff4e408a2c4dfb3a7479c4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/7ce5c23502ff4e408a2c4dfb3a7479c4 2024-12-03T05:16:19,782 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b77bcb7a8a34ed49d9372aa03cd71d1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/8b77bcb7a8a34ed49d9372aa03cd71d1 2024-12-03T05:16:19,783 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/fff4e2f347a145a1bf6d39dd85a0edae to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/C/fff4e2f347a145a1bf6d39dd85a0edae 2024-12-03T05:16:19,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/recovered.edits/717.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b/recovered.edits/717.seqid 2024-12-03T05:16:19,787 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/337aee72174b5cc395d13e96dd50849b 2024-12-03T05:16:19,787 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:16:19,790 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,794 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:16:19,796 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:16:19,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:16:19,798 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733202979798"}]},"ts":"9223372036854775807"} 2024-12-03T05:16:19,801 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:16:19,801 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 337aee72174b5cc395d13e96dd50849b, NAME => 'TestAcidGuarantees,,1733202952554.337aee72174b5cc395d13e96dd50849b.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:16:19,801 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:16:19,801 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733202979801"}]},"ts":"9223372036854775807"} 2024-12-03T05:16:19,803 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:16:19,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-03T05:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-03T05:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:16:19,871 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:19,878 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-443467210_22 at /127.0.0.1:47206 [Waiting for operation #206] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1264088159_22 at /127.0.0.1:49428 [Waiting for operation #364] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=774 (was 714) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2440 (was 2897) 2024-12-03T05:16:19,885 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=774, ProcessCount=11, AvailableMemoryMB=2440 2024-12-03T05:16:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:16:19,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:16:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:19,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:16:19,888 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:19,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-03T05:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:19,889 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:16:19,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742175_1351 (size=1061) 2024-12-03T05:16:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:20,298 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:16:20,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742176_1352 (size=53) 2024-12-03T05:16:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8a02ea06b7843bb222ff704bbe29f0ab, disabling compactions & flushes 2024-12-03T05:16:20,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. after waiting 0 ms 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:20,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:20,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a02ea06b7843bb222ff704bbe29f0ab: Waiting for close lock at 1733202980708Disabling compacts and flushes for region at 1733202980708Disabling writes for close at 1733202980708Writing region close event to WAL at 1733202980708Closed at 1733202980708 2024-12-03T05:16:20,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:16:20,710 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733202980710"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733202980710"}]},"ts":"1733202980710"} 2024-12-03T05:16:20,713 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:16:20,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:16:20,715 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202980715"}]},"ts":"1733202980715"} 2024-12-03T05:16:20,717 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:16:20,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, ASSIGN}] 2024-12-03T05:16:20,718 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, ASSIGN 2024-12-03T05:16:20,719 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:16:20,870 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:20,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, ASSIGN because future has completed 2024-12-03T05:16:20,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:21,031 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:21,031 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:16:21,032 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,032 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:16:21,032 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,032 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,033 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,034 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:21,034 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName A 2024-12-03T05:16:21,035 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:21,040 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:21,040 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,041 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:21,042 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName B 2024-12-03T05:16:21,042 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:21,042 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:21,042 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,043 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:21,044 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName C 2024-12-03T05:16:21,044 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:21,044 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:21,044 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:21,044 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,045 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,045 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,046 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,046 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,046 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:16:21,047 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,049 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:16:21,049 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 8a02ea06b7843bb222ff704bbe29f0ab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71400658, jitterRate=0.06395271420478821}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:16:21,049 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:21,050 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 8a02ea06b7843bb222ff704bbe29f0ab: Running coprocessor pre-open hook at 1733202981032Writing region info on filesystem at 1733202981032Initializing all the Stores at 1733202981033 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202981033Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202981033Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202981033Cleaning up temporary data from old regions at 1733202981046 (+13 ms)Running coprocessor post-open hooks at 1733202981049 (+3 ms)Region opened successfully at 1733202981050 (+1 ms) 2024-12-03T05:16:21,050 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., pid=93, masterSystemTime=1733202981028 2024-12-03T05:16:21,052 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:21,052 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:21,052 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:21,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:21,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-03T05:16:21,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 in 181 msec 2024-12-03T05:16:21,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-03T05:16:21,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, ASSIGN in 339 msec 2024-12-03T05:16:21,059 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:16:21,060 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733202981059"}]},"ts":"1733202981059"} 2024-12-03T05:16:21,061 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:16:21,062 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:16:21,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-03T05:16:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T05:16:22,041 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:22,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce76000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:22,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:22,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:22,043 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc36677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:22,046 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69af49e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:22,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:22,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:22,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:16:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:16:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4669ce9 2024-12-03T05:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:16:22,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:16:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:16:22,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:16:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:22,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742177_1353 (size=1097) 2024-12-03T05:16:22,476 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T05:16:22,476 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T05:16:22,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:16:22,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE}] 2024-12-03T05:16:22,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE 2024-12-03T05:16:22,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:22,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE because future has completed 2024-12-03T05:16:22,486 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:16:22,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:22,639 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,639 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:16:22,639 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 8a02ea06b7843bb222ff704bbe29f0ab, disabling compactions & flushes 2024-12-03T05:16:22,639 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,639 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,639 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. after waiting 0 ms 2024-12-03T05:16:22,639 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,643 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:16:22,644 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,644 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 8a02ea06b7843bb222ff704bbe29f0ab: Waiting for close lock at 1733202982639Running coprocessor pre-close hooks at 1733202982639Disabling compacts and flushes for region at 1733202982639Disabling writes for close at 1733202982639Writing region close event to WAL at 1733202982640 (+1 ms)Running coprocessor post-close hooks at 1733202982644 (+4 ms)Closed at 1733202982644 2024-12-03T05:16:22,644 WARN [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 8a02ea06b7843bb222ff704bbe29f0ab to self. 2024-12-03T05:16:22,646 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,646 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=CLOSED 2024-12-03T05:16:22,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:22,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-03T05:16:22,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 in 163 msec 2024-12-03T05:16:22,652 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE; state=CLOSED, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=true 2024-12-03T05:16:22,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:22,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE because future has completed 2024-12-03T05:16:22,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:22,962 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,962 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:16:22,962 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,963 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:16:22,963 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,963 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,964 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,966 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:22,966 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName A 2024-12-03T05:16:22,967 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:22,967 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:22,968 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,985 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:22,985 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName B 2024-12-03T05:16:22,985 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:22,986 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:22,986 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,988 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:22,988 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a02ea06b7843bb222ff704bbe29f0ab columnFamilyName C 2024-12-03T05:16:22,988 DEBUG [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:22,989 INFO [StoreOpener-8a02ea06b7843bb222ff704bbe29f0ab-1 {}] regionserver.HStore(327): Store=8a02ea06b7843bb222ff704bbe29f0ab/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:22,989 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,989 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,990 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,991 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,992 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,992 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,992 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:16:22,994 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,995 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 8a02ea06b7843bb222ff704bbe29f0ab; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68470290, jitterRate=0.02028682827949524}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:16:22,995 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:22,996 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 8a02ea06b7843bb222ff704bbe29f0ab: Running coprocessor pre-open hook at 1733202982963Writing region info on filesystem at 1733202982963Initializing all the Stores at 1733202982964 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202982964Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202982964Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733202982964Cleaning up temporary data from old regions at 1733202982992 (+28 ms)Running coprocessor post-open hooks at 1733202982995 (+3 ms)Region opened successfully at 1733202982996 (+1 ms) 2024-12-03T05:16:22,997 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., pid=98, masterSystemTime=1733202982956 2024-12-03T05:16:22,998 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,999 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:22,999 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=OPEN, openSeqNum=5, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:23,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-03T05:16:23,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 in 199 msec 2024-12-03T05:16:23,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T05:16:23,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, REOPEN/MOVE in 523 msec 2024-12-03T05:16:23,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-03T05:16:23,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-12-03T05:16:23,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 960 msec 2024-12-03T05:16:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-03T05:16:23,024 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:23,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9088e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,026 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c094db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,027 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282db270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b81bdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606543be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2786927a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3548d87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,035 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5851fef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef0482d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b60a361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8ec14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e62506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0b8a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,043 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1608c113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,046 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe6954e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8e9c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,055 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,057 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa15fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,060 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2141fa68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3822df8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b68c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,066 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577ca645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,069 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6d0b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390f707c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,076 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b3fad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,079 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2eb863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ddbcc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:23,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:23,085 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d71a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:23,088 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd28f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:23,093 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,096 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,099 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,100 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,102 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T05:16:23,103 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:23,106 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,107 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,108 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,109 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,109 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,110 DEBUG [Thread-1616 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,111 DEBUG [Thread-1616 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:23,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:16:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:23,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:23,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:23,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203caf381510e49407782ea72dfcaeb1478_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202983123/Put/seqid=0 2024-12-03T05:16:23,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202983150, max timestamp=1733202983161Num uniques 859; 2024-12-03T05:16:23,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:23,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202983150, max timestamp=1733202983161Num uniques 859; 2024-12-03T05:16:23,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:23,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202983150, max timestamp=1733202983161Num uniques 859; 2024-12-03T05:16:23,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742178_1354 (size=12154) 2024-12-03T05:16:23,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:23,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203caf381510e49407782ea72dfcaeb1478_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caf381510e49407782ea72dfcaeb1478_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:23,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3bf4137a30794f2499799771d85875b6, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:23,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3bf4137a30794f2499799771d85875b6 is 175, key is test_row_0/A:col10/1733202983123/Put/seqid=0 2024-12-03T05:16:23,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742179_1355 (size=30955) 2024-12-03T05:16:23,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3bf4137a30794f2499799771d85875b6 2024-12-03T05:16:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:23,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203043205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203043206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203043209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203043214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203043215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/87b8ba4192984d26aab81ddf400a1ba0 is 50, key is test_row_0/B:col10/1733202983123/Put/seqid=0 2024-12-03T05:16:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742180_1356 (size=12001) 2024-12-03T05:16:23,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:16:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203043330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:23,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:16:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203043541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203043541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203043541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203043541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203043542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:16:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/87b8ba4192984d26aab81ddf400a1ba0 2024-12-03T05:16:23,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/c32ad39400114af38b4b793544179b5a is 50, key is test_row_0/C:col10/1733202983123/Put/seqid=0 2024-12-03T05:16:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:23,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742181_1357 (size=12001) 2024-12-03T05:16:23,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/c32ad39400114af38b4b793544179b5a 2024-12-03T05:16:23,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:16:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3bf4137a30794f2499799771d85875b6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6 2024-12-03T05:16:23,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6, entries=150, sequenceid=15, filesize=30.2 K 2024-12-03T05:16:23,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/87b8ba4192984d26aab81ddf400a1ba0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0 2024-12-03T05:16:23,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T05:16:23,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/c32ad39400114af38b4b793544179b5a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a 2024-12-03T05:16:23,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203043851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203043852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203043852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203043856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203043856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:23,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T05:16:23,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 8a02ea06b7843bb222ff704bbe29f0ab in 760ms, sequenceid=15, compaction requested=false 2024-12-03T05:16:23,884 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T05:16:23,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:23,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T05:16:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:23,909 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:23,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d15d3a4b4f264389926deba4158f7d3a_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202983204/Put/seqid=0 2024-12-03T05:16:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742182_1358 (size=12154) 2024-12-03T05:16:23,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:23,978 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d15d3a4b4f264389926deba4158f7d3a_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d15d3a4b4f264389926deba4158f7d3a_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:23,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c48e22c7d88c4bf5b0e57c48038e64f9, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c48e22c7d88c4bf5b0e57c48038e64f9 is 175, key is test_row_0/A:col10/1733202983204/Put/seqid=0 2024-12-03T05:16:24,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742183_1359 (size=30955) 2024-12-03T05:16:24,042 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c48e22c7d88c4bf5b0e57c48038e64f9 2024-12-03T05:16:24,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/527efb34c6c349d2ad134c56d519ea05 is 50, key is test_row_0/B:col10/1733202983204/Put/seqid=0 2024-12-03T05:16:24,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742184_1360 (size=12001) 2024-12-03T05:16:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:24,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203044396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203044397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203044411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203044414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203044424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,524 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/527efb34c6c349d2ad134c56d519ea05 2024-12-03T05:16:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203044522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203044522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203044533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203044533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203044550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/bade5b43b38544c4b22dc06df410b06e is 50, key is test_row_0/C:col10/1733202983204/Put/seqid=0 2024-12-03T05:16:24,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742185_1361 (size=12001) 2024-12-03T05:16:24,648 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/bade5b43b38544c4b22dc06df410b06e 2024-12-03T05:16:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c48e22c7d88c4bf5b0e57c48038e64f9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9 2024-12-03T05:16:24,684 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9, entries=150, sequenceid=52, filesize=30.2 K 2024-12-03T05:16:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/527efb34c6c349d2ad134c56d519ea05 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05 2024-12-03T05:16:24,692 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05, entries=150, sequenceid=52, filesize=11.7 K 2024-12-03T05:16:24,696 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:16:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/bade5b43b38544c4b22dc06df410b06e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e 2024-12-03T05:16:24,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e, entries=150, sequenceid=52, filesize=11.7 K 2024-12-03T05:16:24,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 8a02ea06b7843bb222ff704bbe29f0ab in 815ms, sequenceid=52, compaction requested=false 2024-12-03T05:16:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T05:16:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T05:16:24,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T05:16:24,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6200 sec 2024-12-03T05:16:24,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6380 sec 2024-12-03T05:16:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:24,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:24,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031452ae54034e4d64af27fb7dfb930b1d_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203044804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742186_1362 (size=14594) 2024-12-03T05:16:24,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203044812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031452ae54034e4d64af27fb7dfb930b1d_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031452ae54034e4d64af27fb7dfb930b1d_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/fb876b87209c4b9eadce709e6a5bf7d3, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:24,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/fb876b87209c4b9eadce709e6a5bf7d3 is 175, key is test_row_0/A:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:24,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203044824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203044824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742187_1363 (size=39549) 2024-12-03T05:16:24,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/fb876b87209c4b9eadce709e6a5bf7d3 2024-12-03T05:16:24,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e486843548c5417b9ee87977cd89bab9 is 50, key is test_row_0/B:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742188_1364 (size=12001) 2024-12-03T05:16:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203044931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203044940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203044940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203044950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203044950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203045151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203045161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203045161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203045171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203045171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T05:16:25,270 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:25,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T05:16:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:25,286 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:25,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:25,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:25,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e486843548c5417b9ee87977cd89bab9 2024-12-03T05:16:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:25,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/7e9a8d57134a44da872e3d92894e543e is 50, key is test_row_0/C:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:25,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742189_1365 (size=12001) 2024-12-03T05:16:25,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/7e9a8d57134a44da872e3d92894e543e 2024-12-03T05:16:25,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/fb876b87209c4b9eadce709e6a5bf7d3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3 2024-12-03T05:16:25,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3, entries=200, sequenceid=64, filesize=38.6 K 2024-12-03T05:16:25,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e486843548c5417b9ee87977cd89bab9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9 2024-12-03T05:16:25,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T05:16:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/7e9a8d57134a44da872e3d92894e543e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e 2024-12-03T05:16:25,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T05:16:25,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a02ea06b7843bb222ff704bbe29f0ab in 726ms, sequenceid=64, compaction requested=true 2024-12-03T05:16:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:25,483 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:25,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:25,483 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:25,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:25,484 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:25,484 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:25,484 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,484 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,484 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.2 K 2024-12-03T05:16:25,484 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=99.1 K 2024-12-03T05:16:25,484 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,484 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3] 2024-12-03T05:16:25,484 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf4137a30794f2499799771d85875b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733202983107 2024-12-03T05:16:25,484 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b8ba4192984d26aab81ddf400a1ba0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733202983107 2024-12-03T05:16:25,485 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c48e22c7d88c4bf5b0e57c48038e64f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733202983204 2024-12-03T05:16:25,485 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 527efb34c6c349d2ad134c56d519ea05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733202983204 2024-12-03T05:16:25,486 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e486843548c5417b9ee87977cd89bab9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:25,486 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb876b87209c4b9eadce709e6a5bf7d3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:25,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202984800, max timestamp=1733202985489Num uniques 859; 2024-12-03T05:16:25,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202984800, max timestamp=1733202985489Num uniques 859; 2024-12-03T05:16:25,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:25,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:25,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202984800, max timestamp=1733202985489Num uniques 859; 2024-12-03T05:16:25,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:25,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:25,525 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:25,531 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120331e183bde5624c28b15044ce9b7e6124_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:25,533 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120331e183bde5624c28b15044ce9b7e6124_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:25,533 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120331e183bde5624c28b15044ce9b7e6124_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:25,540 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#310 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:25,541 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f736b2b9e600466988ae971dbe7156d8 is 50, key is test_row_0/B:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:25,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034d635712219e418c9cd4b34ccad9c99d_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202985508/Put/seqid=0 2024-12-03T05:16:25,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202985576, max timestamp=1733202985594Num uniques 859; 2024-12-03T05:16:25,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:25,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202985576, max timestamp=1733202985594Num uniques 859; 2024-12-03T05:16:25,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:25,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733202985576, max timestamp=1733202985594Num uniques 859; 2024-12-03T05:16:25,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:25,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742190_1366 (size=4469) 2024-12-03T05:16:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742191_1367 (size=12104) 2024-12-03T05:16:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742192_1368 (size=17034) 2024-12-03T05:16:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203045643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203045645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203045647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203045668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203045670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203045781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203045791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203045791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203045803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203045803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:25,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:25,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,020 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#309 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:26,021 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/b187e25769dd4391bb9307a576d908c7 is 175, key is test_row_0/A:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:26,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203046000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203046023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203046023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203046023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203046023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742193_1369 (size=31058) 2024-12-03T05:16:26,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:26,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034d635712219e418c9cd4b34ccad9c99d_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d635712219e418c9cd4b34ccad9c99d_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:26,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/0909e19ba5cf44e884736ba35bf1e190, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:26,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/0909e19ba5cf44e884736ba35bf1e190 is 175, key is test_row_0/A:col10/1733202985508/Put/seqid=0 2024-12-03T05:16:26,076 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f736b2b9e600466988ae971dbe7156d8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f736b2b9e600466988ae971dbe7156d8 2024-12-03T05:16:26,093 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into f736b2b9e600466988ae971dbe7156d8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:26,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:26,093 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733202985483; duration=0sec 2024-12-03T05:16:26,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:26,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:26,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:26,094 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:26,095 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:26,095 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,095 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.2 K 2024-12-03T05:16:26,095 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c32ad39400114af38b4b793544179b5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733202983107 2024-12-03T05:16:26,095 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bade5b43b38544c4b22dc06df410b06e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733202983204 2024-12-03T05:16:26,096 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9a8d57134a44da872e3d92894e543e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742194_1370 (size=48139) 2024-12-03T05:16:26,108 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:26,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/0909e19ba5cf44e884736ba35bf1e190 2024-12-03T05:16:26,109 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/5cb123c3b58a4a5483e4aa0670a009fd is 50, key is test_row_0/C:col10/1733202984401/Put/seqid=0 2024-12-03T05:16:26,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/224dcf01808948849a2c4bfd605b3762 is 50, key is test_row_0/B:col10/1733202985508/Put/seqid=0 2024-12-03T05:16:26,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742195_1371 (size=12104) 2024-12-03T05:16:26,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742196_1372 (size=12001) 2024-12-03T05:16:26,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/224dcf01808948849a2c4bfd605b3762 2024-12-03T05:16:26,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/451aa0aaddf44dc3a26a95653c8a1616 is 50, key is test_row_0/C:col10/1733202985508/Put/seqid=0 2024-12-03T05:16:26,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742197_1373 (size=12001) 2024-12-03T05:16:26,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/451aa0aaddf44dc3a26a95653c8a1616 2024-12-03T05:16:26,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/0909e19ba5cf44e884736ba35bf1e190 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190 2024-12-03T05:16:26,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190, entries=250, sequenceid=89, filesize=47.0 K 2024-12-03T05:16:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/224dcf01808948849a2c4bfd605b3762 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762 2024-12-03T05:16:26,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762, entries=150, sequenceid=89, filesize=11.7 K 2024-12-03T05:16:26,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/451aa0aaddf44dc3a26a95653c8a1616 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616 2024-12-03T05:16:26,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616, entries=150, sequenceid=89, filesize=11.7 K 2024-12-03T05:16:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 8a02ea06b7843bb222ff704bbe29f0ab in 761ms, sequenceid=89, compaction requested=false 2024-12-03T05:16:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:26,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-03T05:16:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:26,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:26,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031e8ae975f4e149e39c2842252228f62f_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203046358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203046360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203046362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203046365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203046382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742198_1374 (size=14594) 2024-12-03T05:16:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:26,461 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/b187e25769dd4391bb9307a576d908c7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7 2024-12-03T05:16:26,466 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into b187e25769dd4391bb9307a576d908c7(size=30.3 K), total size for store is 77.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:26,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:26,466 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733202985483; duration=0sec 2024-12-03T05:16:26,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:26,466 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:26,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203046491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203046491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203046492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203046492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203046511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,550 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/5cb123c3b58a4a5483e4aa0670a009fd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/5cb123c3b58a4a5483e4aa0670a009fd 2024-12-03T05:16:26,565 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 5cb123c3b58a4a5483e4aa0670a009fd(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:26,566 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:26,566 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733202985483; duration=0sec 2024-12-03T05:16:26,566 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:26,566 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:26,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203046737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203046737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203046737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203046737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203046737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:26,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:26,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031e8ae975f4e149e39c2842252228f62f_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e8ae975f4e149e39c2842252228f62f_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:26,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c18f57835d1646068a25e4fd234c7943, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:26,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c18f57835d1646068a25e4fd234c7943 is 175, key is test_row_0/A:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:26,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742199_1375 (size=39549) 2024-12-03T05:16:26,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:26,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203047064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203047064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203047065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203047065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203047065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c18f57835d1646068a25e4fd234c7943 2024-12-03T05:16:27,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/7d4d9d81c3f942178ba52ab30ba492d6 is 50, key is test_row_0/B:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742200_1376 (size=12001) 2024-12-03T05:16:27,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/7d4d9d81c3f942178ba52ab30ba492d6 2024-12-03T05:16:27,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d6b325b3eac04abca23cff7aaad130ea is 50, key is test_row_0/C:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:27,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742201_1377 (size=12001) 2024-12-03T05:16:27,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:27,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d6b325b3eac04abca23cff7aaad130ea 2024-12-03T05:16:27,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:27,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c18f57835d1646068a25e4fd234c7943 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943 2024-12-03T05:16:27,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943, entries=200, sequenceid=125, filesize=38.6 K 2024-12-03T05:16:27,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/7d4d9d81c3f942178ba52ab30ba492d6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6 2024-12-03T05:16:27,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T05:16:27,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d6b325b3eac04abca23cff7aaad130ea as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea 2024-12-03T05:16:27,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T05:16:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:27,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 8a02ea06b7843bb222ff704bbe29f0ab in 1094ms, sequenceid=125, compaction requested=true 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:27,441 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:27,441 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:27,444 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:27,445 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:27,445 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,445 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f736b2b9e600466988ae971dbe7156d8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.3 K 2024-12-03T05:16:27,445 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=116.0 K 2024-12-03T05:16:27,445 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943] 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f736b2b9e600466988ae971dbe7156d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b187e25769dd4391bb9307a576d908c7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:27,445 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 224dcf01808948849a2c4bfd605b3762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733202984800 2024-12-03T05:16:27,446 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0909e19ba5cf44e884736ba35bf1e190, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733202984800 2024-12-03T05:16:27,446 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4d9d81c3f942178ba52ab30ba492d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:27,446 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18f57835d1646068a25e4fd234c7943, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:27,458 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:27,458 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e4dfa9ab86434210986729584eb99299 is 50, key is test_row_0/B:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:27,460 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:27,462 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120365d9bb1e2fa44dbeadf90a835db48bfb_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:27,464 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120365d9bb1e2fa44dbeadf90a835db48bfb_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:27,464 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120365d9bb1e2fa44dbeadf90a835db48bfb_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742202_1378 (size=12207) 2024-12-03T05:16:27,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742203_1379 (size=4469) 2024-12-03T05:16:27,529 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#319 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:27,529 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c5bfc7f79ce84401a8cf8f039cd22fcd is 175, key is test_row_0/A:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:27,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T05:16:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,530 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:27,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:27,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:27,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:27,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:27,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:27,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742204_1380 (size=31161) 2024-12-03T05:16:27,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120314ff2eb6e9674c51a3f60082c2368cdd_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202986376/Put/seqid=0 2024-12-03T05:16:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:27,621 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c5bfc7f79ce84401a8cf8f039cd22fcd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd 2024-12-03T05:16:27,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742205_1381 (size=12304) 2024-12-03T05:16:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:27,632 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120314ff2eb6e9674c51a3f60082c2368cdd_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314ff2eb6e9674c51a3f60082c2368cdd_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:27,633 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into c5bfc7f79ce84401a8cf8f039cd22fcd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:27,633 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:27,633 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733202987441; duration=0sec 2024-12-03T05:16:27,633 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:27,633 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:27,633 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:27,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3818eaf10292444ba17e93d87ea7721e, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:27,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3818eaf10292444ba17e93d87ea7721e is 175, key is test_row_0/A:col10/1733202986376/Put/seqid=0 2024-12-03T05:16:27,642 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:27,642 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:27,642 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:27,643 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/5cb123c3b58a4a5483e4aa0670a009fd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.3 K 2024-12-03T05:16:27,643 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb123c3b58a4a5483e4aa0670a009fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733202984373 2024-12-03T05:16:27,643 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 451aa0aaddf44dc3a26a95653c8a1616, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733202984800 2024-12-03T05:16:27,643 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b325b3eac04abca23cff7aaad130ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742206_1382 (size=31105) 2024-12-03T05:16:27,667 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#321 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:27,668 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3a1916c948ae4480ae3997d5ae5b272b is 50, key is test_row_0/C:col10/1733202985662/Put/seqid=0 2024-12-03T05:16:27,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202987652, max timestamp=1733202987681Num uniques 859; 2024-12-03T05:16:27,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:27,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202987652, max timestamp=1733202987681Num uniques 859; 2024-12-03T05:16:27,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:27,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733202987652, max timestamp=1733202987681Num uniques 859; 2024-12-03T05:16:27,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742207_1383 (size=12207) 2024-12-03T05:16:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203047725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203047728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203047727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203047730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203047737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203047850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203047851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203047861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203047861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203047870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:27,920 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e4dfa9ab86434210986729584eb99299 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e4dfa9ab86434210986729584eb99299 2024-12-03T05:16:27,931 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into e4dfa9ab86434210986729584eb99299(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:27,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:27,931 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733202987441; duration=0sec 2024-12-03T05:16:27,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:27,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:28,059 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3818eaf10292444ba17e93d87ea7721e 2024-12-03T05:16:28,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e48d042bdd894162900c96f1585ed405 is 50, key is test_row_0/B:col10/1733202986376/Put/seqid=0 2024-12-03T05:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203048083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203048083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203048083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,115 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3a1916c948ae4480ae3997d5ae5b272b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3a1916c948ae4480ae3997d5ae5b272b 2024-12-03T05:16:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203048096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203048098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,124 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 3a1916c948ae4480ae3997d5ae5b272b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:28,124 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:28,124 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733202987441; duration=0sec 2024-12-03T05:16:28,124 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:28,124 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742208_1384 (size=12151) 2024-12-03T05:16:28,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203048400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203048411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203048411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203048435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203048435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,544 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e48d042bdd894162900c96f1585ed405 2024-12-03T05:16:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3951881c618849f6b715fe5088e7ebc1 is 50, key is test_row_0/C:col10/1733202986376/Put/seqid=0 2024-12-03T05:16:28,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742209_1385 (size=12151) 2024-12-03T05:16:28,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3951881c618849f6b715fe5088e7ebc1 2024-12-03T05:16:28,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/3818eaf10292444ba17e93d87ea7721e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e 2024-12-03T05:16:28,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e, entries=150, sequenceid=137, filesize=30.4 K 2024-12-03T05:16:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e48d042bdd894162900c96f1585ed405 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405 2024-12-03T05:16:28,630 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405, entries=150, sequenceid=137, filesize=11.9 K 2024-12-03T05:16:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3951881c618849f6b715fe5088e7ebc1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1 2024-12-03T05:16:28,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1, entries=150, sequenceid=137, filesize=11.9 K 2024-12-03T05:16:28,644 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a02ea06b7843bb222ff704bbe29f0ab in 1114ms, sequenceid=137, compaction requested=false 2024-12-03T05:16:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T05:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T05:16:28,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.3770 sec 2024-12-03T05:16:28,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T05:16:28,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3590 sec 2024-12-03T05:16:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:28,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:28,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393cafe83c453485a96077160dd13614e_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203048952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203048967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203048968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203048971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203048971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:28,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742210_1386 (size=14794) 2024-12-03T05:16:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203049070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203049090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203049091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203049291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203049301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203049310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:29,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393cafe83c453485a96077160dd13614e_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393cafe83c453485a96077160dd13614e_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:29,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/7f98b35bf5a54d8f87de5bcbf46f23e5, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:29,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/7f98b35bf5a54d8f87de5bcbf46f23e5 is 175, key is test_row_0/A:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T05:16:29,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:29,454 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T05:16:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:29,456 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:29,457 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:29,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:29,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742211_1387 (size=39749) 2024-12-03T05:16:29,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/7f98b35bf5a54d8f87de5bcbf46f23e5 2024-12-03T05:16:29,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/ea9f727d4b0f422c82791f536e122b18 is 50, key is test_row_0/B:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:29,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742212_1388 (size=12151) 2024-12-03T05:16:29,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:29,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:29,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203049610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203049621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203049630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:29,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:29,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:29,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/ea9f727d4b0f422c82791f536e122b18 2024-12-03T05:16:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203049992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,003 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/185110bb45e24c66a0ed631d9a035671 is 50, key is test_row_0/C:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203049996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,010 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742213_1389 (size=12151) 2024-12-03T05:16:30,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/185110bb45e24c66a0ed631d9a035671 2024-12-03T05:16:30,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/7f98b35bf5a54d8f87de5bcbf46f23e5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5 2024-12-03T05:16:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:30,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5, entries=200, sequenceid=177, filesize=38.8 K 2024-12-03T05:16:30,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/ea9f727d4b0f422c82791f536e122b18 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18 2024-12-03T05:16:30,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T05:16:30,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/185110bb45e24c66a0ed631d9a035671 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671 2024-12-03T05:16:30,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T05:16:30,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8a02ea06b7843bb222ff704bbe29f0ab in 1189ms, sequenceid=177, compaction requested=true 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T05:16:30,127 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:30,128 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:30,129 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:30,129 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:30,129 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,129 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=99.6 K 2024-12-03T05:16:30,129 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,129 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5] 2024-12-03T05:16:30,133 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bfc7f79ce84401a8cf8f039cd22fcd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:30,142 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3818eaf10292444ba17e93d87ea7721e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733202986354 2024-12-03T05:16:30,142 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:30,142 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:30,142 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,143 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3a1916c948ae4480ae3997d5ae5b272b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.7 K 2024-12-03T05:16:30,144 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f98b35bf5a54d8f87de5bcbf46f23e5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987710 2024-12-03T05:16:30,145 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1916c948ae4480ae3997d5ae5b272b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:30,150 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3951881c618849f6b715fe5088e7ebc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733202986354 2024-12-03T05:16:30,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:30,159 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185110bb45e24c66a0ed631d9a035671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987725 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:30,201 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b15d8164726b41b2a31361ebc1d903a1_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202988946/Put/seqid=0 2024-12-03T05:16:30,234 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:30,235 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/a54eb94c59dd4f00a4ef2786a260ab83 is 50, key is test_row_0/C:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:30,241 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033deb5a3c3f3c4040b93db5cfb446a45f_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,243 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033deb5a3c3f3c4040b93db5cfb446a45f_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,243 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033deb5a3c3f3c4040b93db5cfb446a45f_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742214_1390 (size=12304) 2024-12-03T05:16:30,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:30,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b15d8164726b41b2a31361ebc1d903a1_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b15d8164726b41b2a31361ebc1d903a1_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:30,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742215_1391 (size=12459) 2024-12-03T05:16:30,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ad5d88b9ead4470a9ac667fadd780682, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ad5d88b9ead4470a9ac667fadd780682 is 175, key is test_row_0/A:col10/1733202988946/Put/seqid=0 2024-12-03T05:16:30,292 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/a54eb94c59dd4f00a4ef2786a260ab83 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/a54eb94c59dd4f00a4ef2786a260ab83 2024-12-03T05:16:30,298 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into a54eb94c59dd4f00a4ef2786a260ab83(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:30,298 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:30,299 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733202990127; duration=0sec 2024-12-03T05:16:30,299 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:30,299 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:30,299 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:30,300 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:30,300 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:30,300 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,300 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e4dfa9ab86434210986729584eb99299, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.7 K 2024-12-03T05:16:30,300 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4dfa9ab86434210986729584eb99299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733202985628 2024-12-03T05:16:30,301 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48d042bdd894162900c96f1585ed405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733202986354 2024-12-03T05:16:30,305 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea9f727d4b0f422c82791f536e122b18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987725 2024-12-03T05:16:30,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202990271, max timestamp=1733202990304Num uniques 859; 2024-12-03T05:16:30,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:30,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202990271, max timestamp=1733202990304Num uniques 859; 2024-12-03T05:16:30,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202990271, max timestamp=1733202990304Num uniques 859; 2024-12-03T05:16:30,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:30,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:30,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742216_1392 (size=4469) 2024-12-03T05:16:30,326 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:30,327 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/3e9906424b194681b7087c31e45b72ad is 50, key is test_row_0/B:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:30,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742217_1393 (size=31105) 2024-12-03T05:16:30,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ad5d88b9ead4470a9ac667fadd780682 2024-12-03T05:16:30,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e433bdf1a22d4f2d886849c5ac58f102 is 50, key is test_row_0/B:col10/1733202988946/Put/seqid=0 2024-12-03T05:16:30,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742218_1394 (size=12459) 2024-12-03T05:16:30,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742219_1395 (size=12151) 2024-12-03T05:16:30,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e433bdf1a22d4f2d886849c5ac58f102 2024-12-03T05:16:30,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/0f57fcfceb0c4212889b8fda0bf7a949 is 50, key is test_row_0/C:col10/1733202988946/Put/seqid=0 2024-12-03T05:16:30,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742220_1396 (size=12151) 2024-12-03T05:16:30,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/0f57fcfceb0c4212889b8fda0bf7a949 2024-12-03T05:16:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203050388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203050388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203050388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ad5d88b9ead4470a9ac667fadd780682 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682 2024-12-03T05:16:30,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682, entries=150, sequenceid=188, filesize=30.4 K 2024-12-03T05:16:30,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e433bdf1a22d4f2d886849c5ac58f102 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102 2024-12-03T05:16:30,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T05:16:30,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/0f57fcfceb0c4212889b8fda0bf7a949 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949 2024-12-03T05:16:30,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T05:16:30,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8a02ea06b7843bb222ff704bbe29f0ab in 285ms, sequenceid=188, compaction requested=false 2024-12-03T05:16:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:30,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:30,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c2e363a740442afaeb39a73993cc379_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:30,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742221_1397 (size=14794) 2024-12-03T05:16:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203050548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203050562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203050563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:30,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203050672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203050681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203050688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,730 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:30,730 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/8325a7fa3e5b4233a51f0caf65017683 is 175, key is test_row_0/A:col10/1733202987725/Put/seqid=0 2024-12-03T05:16:30,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742222_1398 (size=31413) 2024-12-03T05:16:30,796 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/3e9906424b194681b7087c31e45b72ad as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3e9906424b194681b7087c31e45b72ad 2024-12-03T05:16:30,811 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/8325a7fa3e5b4233a51f0caf65017683 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683 2024-12-03T05:16:30,823 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into 3e9906424b194681b7087c31e45b72ad(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:30,823 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:30,823 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733202990127; duration=0sec 2024-12-03T05:16:30,823 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:30,823 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:30,824 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into 8325a7fa3e5b4233a51f0caf65017683(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:30,824 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:30,824 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733202990127; duration=0sec 2024-12-03T05:16:30,824 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:30,824 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:30,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:30,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203050900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203050900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203050912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:30,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:30,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c2e363a740442afaeb39a73993cc379_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2e363a740442afaeb39a73993cc379_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:30,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1424f91e23b4b739f5dab1d6ba68412, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:30,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1424f91e23b4b739f5dab1d6ba68412 is 175, key is test_row_0/A:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742223_1399 (size=39749) 2024-12-03T05:16:31,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1424f91e23b4b739f5dab1d6ba68412 2024-12-03T05:16:31,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/38a5d75575c34006b39032e4fb650ec5 is 50, key is test_row_0/B:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:31,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742224_1400 (size=12151) 2024-12-03T05:16:31,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203051221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203051221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203051231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/38a5d75575c34006b39032e4fb650ec5 2024-12-03T05:16:31,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/422f5c81faad438aab88dd1724ae91d3 is 50, key is test_row_0/C:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:31,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742225_1401 (size=12151) 2024-12-03T05:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:31,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203051740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203051740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203051763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:31,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/422f5c81faad438aab88dd1724ae91d3 2024-12-03T05:16:31,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:31,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1424f91e23b4b739f5dab1d6ba68412 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412 2024-12-03T05:16:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:31,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:31,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412, entries=200, sequenceid=226, filesize=38.8 K 2024-12-03T05:16:31,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/38a5d75575c34006b39032e4fb650ec5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5 2024-12-03T05:16:32,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T05:16:32,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/422f5c81faad438aab88dd1724ae91d3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3 2024-12-03T05:16:32,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T05:16:32,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8a02ea06b7843bb222ff704bbe29f0ab in 1491ms, sequenceid=226, compaction requested=true 2024-12-03T05:16:32,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:32,015 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:32,016 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:32,016 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:32,017 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:32,017 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,017 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=99.9 K 2024-12-03T05:16:32,017 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,017 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412] 2024-12-03T05:16:32,017 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:32,017 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:32,017 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,017 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3e9906424b194681b7087c31e45b72ad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.9 K 2024-12-03T05:16:32,018 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8325a7fa3e5b4233a51f0caf65017683, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987725 2024-12-03T05:16:32,018 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9906424b194681b7087c31e45b72ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987725 2024-12-03T05:16:32,018 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5d88b9ead4470a9ac667fadd780682, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733202988946 2024-12-03T05:16:32,018 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e433bdf1a22d4f2d886849c5ac58f102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733202988946 2024-12-03T05:16:32,018 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1424f91e23b4b739f5dab1d6ba68412, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990346 2024-12-03T05:16:32,019 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 38a5d75575c34006b39032e4fb650ec5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990383 2024-12-03T05:16:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:32,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:32,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:32,037 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:32,040 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:32,041 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/941c7047cb1f4214acbd40d04e06bafd is 50, key is test_row_0/B:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:32,043 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f4728364b0144e959ec6e1813dda7879_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:32,046 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f4728364b0144e959ec6e1813dda7879_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:32,046 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f4728364b0144e959ec6e1813dda7879_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:32,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff4a3c4504404c5da5b58782ebe67ef0_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202990560/Put/seqid=0 2024-12-03T05:16:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742226_1402 (size=12561) 2024-12-03T05:16:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742227_1403 (size=4469) 2024-12-03T05:16:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742228_1404 (size=14794) 2024-12-03T05:16:32,121 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/941c7047cb1f4214acbd40d04e06bafd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/941c7047cb1f4214acbd40d04e06bafd 2024-12-03T05:16:32,125 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into 941c7047cb1f4214acbd40d04e06bafd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:32,125 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:32,125 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733202992016; duration=0sec 2024-12-03T05:16:32,125 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:32,125 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:32,126 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:32,126 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:32,127 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:32,127 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,127 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/a54eb94c59dd4f00a4ef2786a260ab83, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=35.9 K 2024-12-03T05:16:32,128 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a54eb94c59dd4f00a4ef2786a260ab83, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733202987725 2024-12-03T05:16:32,128 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f57fcfceb0c4212889b8fda0bf7a949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733202988946 2024-12-03T05:16:32,129 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 422f5c81faad438aab88dd1724ae91d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990383 2024-12-03T05:16:32,135 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:32,136 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/6b3add75404944609b561b76fcd92255 is 50, key is test_row_0/C:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:32,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742229_1405 (size=12561) 2024-12-03T05:16:32,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203052138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202992127, max timestamp=1733202992142Num uniques 859; 2024-12-03T05:16:32,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:32,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202992127, max timestamp=1733202992142Num uniques 859; 2024-12-03T05:16:32,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:32,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202992127, max timestamp=1733202992142Num uniques 859; 2024-12-03T05:16:32,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:32,157 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/6b3add75404944609b561b76fcd92255 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6b3add75404944609b561b76fcd92255 2024-12-03T05:16:32,171 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 6b3add75404944609b561b76fcd92255(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:32,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:32,171 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733202992016; duration=0sec 2024-12-03T05:16:32,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:32,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:32,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203052292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203052293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203052411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203052411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,512 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#336 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:32,513 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/82d96843851f4d33a688d0a5fb487a82 is 175, key is test_row_0/A:col10/1733202990521/Put/seqid=0 2024-12-03T05:16:32,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:32,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff4a3c4504404c5da5b58782ebe67ef0_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff4a3c4504404c5da5b58782ebe67ef0_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:32,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9b0b8eaad92f4884b0e92d4597ba4b5d, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:32,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9b0b8eaad92f4884b0e92d4597ba4b5d is 175, key is test_row_0/A:col10/1733202990560/Put/seqid=0 2024-12-03T05:16:32,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742230_1406 (size=31515) 2024-12-03T05:16:32,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742231_1407 (size=39749) 2024-12-03T05:16:32,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203052631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203052631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203052760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,772 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203052771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,777 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203052790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,801 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:32,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:32,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203052953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203052956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:32,982 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/82d96843851f4d33a688d0a5fb487a82 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82 2024-12-03T05:16:32,986 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into 82d96843851f4d33a688d0a5fb487a82(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:32,986 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:32,986 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733202992015; duration=0sec 2024-12-03T05:16:32,986 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:32,986 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:32,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9b0b8eaad92f4884b0e92d4597ba4b5d 2024-12-03T05:16:32,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/5943c52a92f849e1a87651728542a11f is 50, key is test_row_0/B:col10/1733202990560/Put/seqid=0 2024-12-03T05:16:33,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742232_1408 (size=12151) 2024-12-03T05:16:33,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/5943c52a92f849e1a87651728542a11f 2024-12-03T05:16:33,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/9e4efe9ff97b439d92e68899d1685286 is 50, key is test_row_0/C:col10/1733202990560/Put/seqid=0 2024-12-03T05:16:33,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:33,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:33,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742233_1409 (size=12151) 2024-12-03T05:16:33,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/9e4efe9ff97b439d92e68899d1685286 2024-12-03T05:16:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9b0b8eaad92f4884b0e92d4597ba4b5d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d 2024-12-03T05:16:33,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d, entries=200, sequenceid=239, filesize=38.8 K 2024-12-03T05:16:33,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/5943c52a92f849e1a87651728542a11f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f 2024-12-03T05:16:33,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T05:16:33,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/9e4efe9ff97b439d92e68899d1685286 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286 2024-12-03T05:16:33,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T05:16:33,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8a02ea06b7843bb222ff704bbe29f0ab in 1123ms, sequenceid=239, compaction requested=false 2024-12-03T05:16:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:33,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T05:16:33,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:33,232 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203030b73c72cb84bbc84a77a52834a1c4c_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202992241/Put/seqid=0 2024-12-03T05:16:33,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742234_1410 (size=12454) 2024-12-03T05:16:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203053518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203053526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203053645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203053648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:33,701 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203030b73c72cb84bbc84a77a52834a1c4c_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203030b73c72cb84bbc84a77a52834a1c4c_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/4fed620375fb4803b74d8933996c5944, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/4fed620375fb4803b74d8933996c5944 is 175, key is test_row_0/A:col10/1733202992241/Put/seqid=0 2024-12-03T05:16:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742235_1411 (size=31255) 2024-12-03T05:16:33,764 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/4fed620375fb4803b74d8933996c5944 2024-12-03T05:16:33,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/3717be82560a4c6cb0abed178dfdbdcf is 50, key is test_row_0/B:col10/1733202992241/Put/seqid=0 2024-12-03T05:16:33,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742236_1412 (size=12301) 2024-12-03T05:16:33,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203053861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203053864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203054180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203054181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,236 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/3717be82560a4c6cb0abed178dfdbdcf 2024-12-03T05:16:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/aeae67b09a0d4cb29eefae88578a8622 is 50, key is test_row_0/C:col10/1733202992241/Put/seqid=0 2024-12-03T05:16:34,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742237_1413 (size=12301) 2024-12-03T05:16:34,252 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/aeae67b09a0d4cb29eefae88578a8622 2024-12-03T05:16:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/4fed620375fb4803b74d8933996c5944 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944 2024-12-03T05:16:34,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944, entries=150, sequenceid=278, filesize=30.5 K 2024-12-03T05:16:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/3717be82560a4c6cb0abed178dfdbdcf as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf 2024-12-03T05:16:34,281 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf, entries=150, sequenceid=278, filesize=12.0 K 2024-12-03T05:16:34,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/aeae67b09a0d4cb29eefae88578a8622 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622 2024-12-03T05:16:34,292 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622, entries=150, sequenceid=278, filesize=12.0 K 2024-12-03T05:16:34,293 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 8a02ea06b7843bb222ff704bbe29f0ab in 1061ms, sequenceid=278, compaction requested=true 2024-12-03T05:16:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T05:16:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T05:16:34,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T05:16:34,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8380 sec 2024-12-03T05:16:34,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 4.8450 sec 2024-12-03T05:16:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:34,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:34,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2a2cdbea55542eaa11a46ff75f75331_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:34,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742238_1414 (size=14994) 2024-12-03T05:16:34,704 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:34,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2a2cdbea55542eaa11a46ff75f75331_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2a2cdbea55542eaa11a46ff75f75331_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:34,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/437f003cb03147a789503c49157e8e51, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:34,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/437f003cb03147a789503c49157e8e51 is 175, key is test_row_0/A:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742239_1415 (size=39949) 2024-12-03T05:16:34,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203054780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,792 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203054785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203054785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203054800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,807 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203054810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,819 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203054900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203054900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/437f003cb03147a789503c49157e8e51 2024-12-03T05:16:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203055120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203055120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/228e8627dce243afa7e088b4dab29aed is 50, key is test_row_0/B:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:35,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742240_1416 (size=12301) 2024-12-03T05:16:35,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/228e8627dce243afa7e088b4dab29aed 2024-12-03T05:16:35,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/63e832e0ca894af2b909aeae36e0d6c5 is 50, key is test_row_0/C:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:35,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742241_1417 (size=12301) 2024-12-03T05:16:35,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/63e832e0ca894af2b909aeae36e0d6c5 2024-12-03T05:16:35,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/437f003cb03147a789503c49157e8e51 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51 2024-12-03T05:16:35,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51, entries=200, sequenceid=290, filesize=39.0 K 2024-12-03T05:16:35,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/228e8627dce243afa7e088b4dab29aed as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed 2024-12-03T05:16:35,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T05:16:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/63e832e0ca894af2b909aeae36e0d6c5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5 2024-12-03T05:16:35,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T05:16:35,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a02ea06b7843bb222ff704bbe29f0ab in 548ms, sequenceid=290, compaction requested=true 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:35,242 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:35,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:16:35,244 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:35,244 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:35,244 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:35,244 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=139.1 K 2024-12-03T05:16:35,244 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:35,244 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51] 2024-12-03T05:16:35,244 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:35,245 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82d96843851f4d33a688d0a5fb487a82, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990383 2024-12-03T05:16:35,247 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0b8eaad92f4884b0e92d4597ba4b5d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733202990545 2024-12-03T05:16:35,248 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fed620375fb4803b74d8933996c5944, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733202992241 2024-12-03T05:16:35,248 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437f003cb03147a789503c49157e8e51, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:35,248 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:35,248 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:35,249 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:35,249 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/941c7047cb1f4214acbd40d04e06bafd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=48.2 K 2024-12-03T05:16:35,249 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 941c7047cb1f4214acbd40d04e06bafd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990383 2024-12-03T05:16:35,250 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5943c52a92f849e1a87651728542a11f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733202990560 2024-12-03T05:16:35,250 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3717be82560a4c6cb0abed178dfdbdcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733202992241 2024-12-03T05:16:35,251 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 228e8627dce243afa7e088b4dab29aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:35,269 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:35,276 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:35,277 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/a35cb0465bd644c7b1a552e57dc4c364 is 50, key is test_row_0/B:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:35,278 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e17bf996a4624215a983f88934e0e86f_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:35,281 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e17bf996a4624215a983f88934e0e86f_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:35,281 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e17bf996a4624215a983f88934e0e86f_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:35,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742242_1418 (size=12847) 2024-12-03T05:16:35,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742243_1419 (size=4469) 2024-12-03T05:16:35,309 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#348 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:35,310 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/da735daee7424b15b68d8c3e1c48586b is 175, key is test_row_0/A:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:35,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742244_1420 (size=31801) 2024-12-03T05:16:35,384 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/da735daee7424b15b68d8c3e1c48586b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b 2024-12-03T05:16:35,392 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into da735daee7424b15b68d8c3e1c48586b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:35,392 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:35,392 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=12, startTime=1733202995242; duration=0sec 2024-12-03T05:16:35,392 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:35,392 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:35,392 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:35,393 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:35,393 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:35,393 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:35,393 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6b3add75404944609b561b76fcd92255, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=48.2 K 2024-12-03T05:16:35,394 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3add75404944609b561b76fcd92255, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733202990383 2024-12-03T05:16:35,394 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4efe9ff97b439d92e68899d1685286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733202990560 2024-12-03T05:16:35,394 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeae67b09a0d4cb29eefae88578a8622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733202992241 2024-12-03T05:16:35,395 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e832e0ca894af2b909aeae36e0d6c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:35,417 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:35,417 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/0bfbfa1f9cac4f2e9dc5b83c4062109e is 50, key is test_row_0/C:col10/1733202993514/Put/seqid=0 2024-12-03T05:16:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:35,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:35,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742245_1421 (size=12847) 2024-12-03T05:16:35,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031b56e334f6374d18b7abc4514523cbf4_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202995444/Put/seqid=0 2024-12-03T05:16:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203055488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203055490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742246_1422 (size=17534) 2024-12-03T05:16:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203055611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203055611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,720 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/a35cb0465bd644c7b1a552e57dc4c364 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/a35cb0465bd644c7b1a552e57dc4c364 2024-12-03T05:16:35,729 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into a35cb0465bd644c7b1a552e57dc4c364(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:35,729 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:35,729 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=12, startTime=1733202995242; duration=0sec 2024-12-03T05:16:35,729 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:35,729 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:35,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203055831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203055831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:35,867 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/0bfbfa1f9cac4f2e9dc5b83c4062109e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0bfbfa1f9cac4f2e9dc5b83c4062109e 2024-12-03T05:16:35,883 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 0bfbfa1f9cac4f2e9dc5b83c4062109e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:35,883 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:35,883 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=12, startTime=1733202995242; duration=0sec 2024-12-03T05:16:35,883 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:35,884 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:35,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:35,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031b56e334f6374d18b7abc4514523cbf4_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031b56e334f6374d18b7abc4514523cbf4_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:35,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c8cd35ce323e4b4da788a6446ab63828, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:35,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c8cd35ce323e4b4da788a6446ab63828 is 175, key is test_row_0/A:col10/1733202995444/Put/seqid=0 2024-12-03T05:16:35,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742247_1423 (size=48639) 2024-12-03T05:16:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203056150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203056152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:36,361 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c8cd35ce323e4b4da788a6446ab63828 2024-12-03T05:16:36,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/028731f950ec469ebc624a4141b7fc71 is 50, key is test_row_0/B:col10/1733202995444/Put/seqid=0 2024-12-03T05:16:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742248_1424 (size=12301) 2024-12-03T05:16:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203056671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:36,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203056671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:36,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/028731f950ec469ebc624a4141b7fc71 2024-12-03T05:16:36,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/394f3edecd68436191a07175b8cb185f is 50, key is test_row_0/C:col10/1733202995444/Put/seqid=0 2024-12-03T05:16:36,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742249_1425 (size=12301) 2024-12-03T05:16:36,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/394f3edecd68436191a07175b8cb185f 2024-12-03T05:16:36,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/c8cd35ce323e4b4da788a6446ab63828 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828 2024-12-03T05:16:36,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828, entries=250, sequenceid=313, filesize=47.5 K 2024-12-03T05:16:36,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/028731f950ec469ebc624a4141b7fc71 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71 2024-12-03T05:16:36,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T05:16:36,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/394f3edecd68436191a07175b8cb185f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f 2024-12-03T05:16:36,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T05:16:36,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 8a02ea06b7843bb222ff704bbe29f0ab in 1427ms, sequenceid=313, compaction requested=false 2024-12-03T05:16:36,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T05:16:37,671 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:37,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-03T05:16:37,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:37,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:37,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:37,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:37,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:37,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:37,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:37,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037ef149cf8c824bc792b9ce21c2f6e2d0_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742250_1426 (size=14994) 2024-12-03T05:16:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:37,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733202997744, max timestamp=1733202997808Num uniques 859; 2024-12-03T05:16:37,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:37,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:37,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:37,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203057832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:37,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203057845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203057950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:37,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203057961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:37,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:37,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:37,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:37,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:37,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:38,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:38,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037ef149cf8c824bc792b9ce21c2f6e2d0_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ef149cf8c824bc792b9ce21c2f6e2d0_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:38,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/69910c3273f748c59e9b50653953eabd, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:38,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/69910c3273f748c59e9b50653953eabd is 175, key is test_row_0/A:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742251_1427 (size=39949) 2024-12-03T05:16:38,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203058161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203058170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:38,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203058482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203058504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/69910c3273f748c59e9b50653953eabd 2024-12-03T05:16:38,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/9734356bc9064aea9528982ff3f2be7e is 50, key is test_row_0/B:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:38,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742252_1428 (size=12301) 2024-12-03T05:16:38,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/9734356bc9064aea9528982ff3f2be7e 2024-12-03T05:16:38,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d7177737550a4e0599b3ac18955f388e is 50, key is test_row_0/C:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:38,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742253_1429 (size=12301) 2024-12-03T05:16:38,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53612 deadline: 1733203058820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,833 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53602 deadline: 1733203058844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,849 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53630 deadline: 1733203058851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:38,864 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 8a02ea06b7843bb222ff704bbe29f0ab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8316 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:16:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:38,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:38,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203059001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:39,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203059010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:39,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d7177737550a4e0599b3ac18955f388e 2024-12-03T05:16:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/69910c3273f748c59e9b50653953eabd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd 2024-12-03T05:16:39,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd, entries=200, sequenceid=326, filesize=39.0 K 2024-12-03T05:16:39,070 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-12-03T05:16:39,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/9734356bc9064aea9528982ff3f2be7e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e 2024-12-03T05:16:39,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T05:16:39,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/d7177737550a4e0599b3ac18955f388e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e 2024-12-03T05:16:39,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e, entries=150, sequenceid=326, filesize=12.0 K 2024-12-03T05:16:39,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 8a02ea06b7843bb222ff704bbe29f0ab in 1392ms, sequenceid=326, compaction requested=true 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:39,087 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:39,087 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:39,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=123.00 KB heapSize=361.72 KB 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:39,093 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:39,093 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:39,093 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:39,093 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:39,093 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=117.6 K 2024-12-03T05:16:39,093 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,093 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,094 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd] 2024-12-03T05:16:39,094 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/a35cb0465bd644c7b1a552e57dc4c364, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=36.6 K 2024-12-03T05:16:39,094 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting da735daee7424b15b68d8c3e1c48586b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:39,094 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cd35ce323e4b4da788a6446ab63828, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733202994758 2024-12-03T05:16:39,097 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69910c3273f748c59e9b50653953eabd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995459 2024-12-03T05:16:39,097 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a35cb0465bd644c7b1a552e57dc4c364, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:39,097 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 028731f950ec469ebc624a4141b7fc71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733202994767 2024-12-03T05:16:39,097 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9734356bc9064aea9528982ff3f2be7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995486 2024-12-03T05:16:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d258237c81484e53b3685391372716fb_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733202997830/Put/seqid=0 2024-12-03T05:16:39,105 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:39,118 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:39,119 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/9baaa0fee1f34cd3a1c51f5440919388 is 50, key is test_row_0/B:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:39,136 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120301ab752683e64f93934947c394be712d_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:39,139 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120301ab752683e64f93934947c394be712d_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:39,139 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120301ab752683e64f93934947c394be712d_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:39,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742254_1430 (size=12454) 2024-12-03T05:16:39,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,193 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d258237c81484e53b3685391372716fb_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d258237c81484e53b3685391372716fb_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9142c8ddf22b45febe16aadff726bc99, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9142c8ddf22b45febe16aadff726bc99 is 175, key is test_row_0/A:col10/1733202997830/Put/seqid=0 2024-12-03T05:16:39,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742256_1432 (size=4469) 2024-12-03T05:16:39,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742255_1431 (size=12949) 2024-12-03T05:16:39,223 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#358 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:39,223 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/920ac2bf77d04ecd9db8d2fa815d9c47 is 175, key is test_row_0/A:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:39,227 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/9baaa0fee1f34cd3a1c51f5440919388 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9baaa0fee1f34cd3a1c51f5440919388 2024-12-03T05:16:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742257_1433 (size=31255) 2024-12-03T05:16:39,237 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into 9baaa0fee1f34cd3a1c51f5440919388(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:39,237 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:39,237 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733202999087; duration=0sec 2024-12-03T05:16:39,237 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:39,237 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:39,237 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:39,237 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9142c8ddf22b45febe16aadff726bc99 2024-12-03T05:16:39,238 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:39,238 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:39,238 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,238 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0bfbfa1f9cac4f2e9dc5b83c4062109e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=36.6 K 2024-12-03T05:16:39,239 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfbfa1f9cac4f2e9dc5b83c4062109e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733202993491 2024-12-03T05:16:39,239 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 394f3edecd68436191a07175b8cb185f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733202994767 2024-12-03T05:16:39,239 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d7177737550a4e0599b3ac18955f388e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995486 2024-12-03T05:16:39,252 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:39,252 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/fa7356acd96f459e939c450c8c1087b9 is 50, key is test_row_0/C:col10/1733202997694/Put/seqid=0 2024-12-03T05:16:39,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742258_1434 (size=31903) 2024-12-03T05:16:39,277 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/920ac2bf77d04ecd9db8d2fa815d9c47 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47 2024-12-03T05:16:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c074fbe86297476a948572454d49df96 is 50, key is test_row_0/B:col10/1733202997830/Put/seqid=0 2024-12-03T05:16:39,284 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into 920ac2bf77d04ecd9db8d2fa815d9c47(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:39,284 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:39,285 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733202999087; duration=0sec 2024-12-03T05:16:39,285 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:39,285 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:39,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742259_1435 (size=12949) 2024-12-03T05:16:39,291 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/fa7356acd96f459e939c450c8c1087b9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/fa7356acd96f459e939c450c8c1087b9 2024-12-03T05:16:39,300 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into fa7356acd96f459e939c450c8c1087b9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:39,300 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:39,300 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733202999087; duration=0sec 2024-12-03T05:16:39,300 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:39,300 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:39,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742260_1436 (size=12301) 2024-12-03T05:16:39,322 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c074fbe86297476a948572454d49df96 2024-12-03T05:16:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3561e682c286422fb10e38a687d6dc67 is 50, key is test_row_0/C:col10/1733202997830/Put/seqid=0 2024-12-03T05:16:39,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742261_1437 (size=12301) 2024-12-03T05:16:39,805 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3561e682c286422fb10e38a687d6dc67 2024-12-03T05:16:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/9142c8ddf22b45febe16aadff726bc99 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99 2024-12-03T05:16:39,822 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99, entries=150, sequenceid=352, filesize=30.5 K 2024-12-03T05:16:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c074fbe86297476a948572454d49df96 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96 2024-12-03T05:16:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,833 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96, entries=150, sequenceid=352, filesize=12.0 K 2024-12-03T05:16:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/3561e682c286422fb10e38a687d6dc67 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67 2024-12-03T05:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,847 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67, entries=150, sequenceid=352, filesize=12.0 K 2024-12-03T05:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~361.67 KB/370352, currentSize=0 B/0 for 8a02ea06b7843bb222ff704bbe29f0ab in 762ms, sequenceid=352, compaction requested=false 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:39,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T05:16:39,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-12-03T05:16:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1840 sec 2024-12-03T05:16:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:40,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:40,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e49526978e144ab18917a305149bd9ae_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742262_1438 (size=22618) 2024-12-03T05:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203000252, max timestamp=1733203000273Num uniques 859; 2024-12-03T05:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203000252, max timestamp=1733203000273Num uniques 859; 2024-12-03T05:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:40,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203000252, max timestamp=1733203000273Num uniques 859; 2024-12-03T05:16:40,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203060352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203060359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203060472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203060481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:40,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e49526978e144ab18917a305149bd9ae_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e49526978e144ab18917a305149bd9ae_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:40,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ed1b700214424e4da5f25f375831eafb, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:40,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ed1b700214424e4da5f25f375831eafb is 175, key is test_row_0/A:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742263_1439 (size=66023) 2024-12-03T05:16:40,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ed1b700214424e4da5f25f375831eafb 2024-12-03T05:16:40,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f37baab9b58e450c8351c7ac01f17d8b is 50, key is test_row_0/B:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742264_1440 (size=12301) 2024-12-03T05:16:40,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f37baab9b58e450c8351c7ac01f17d8b 2024-12-03T05:16:40,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/223ef4a2234247628071b8fc9e20eec6 is 50, key is test_row_0/C:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742265_1441 (size=12301) 2024-12-03T05:16:40,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/223ef4a2234247628071b8fc9e20eec6 2024-12-03T05:16:40,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/ed1b700214424e4da5f25f375831eafb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb 2024-12-03T05:16:40,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb, entries=350, sequenceid=365, filesize=64.5 K 2024-12-03T05:16:40,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f37baab9b58e450c8351c7ac01f17d8b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b 2024-12-03T05:16:40,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T05:16:40,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/223ef4a2234247628071b8fc9e20eec6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6 2024-12-03T05:16:40,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203060690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203060700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:40,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T05:16:40,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a02ea06b7843bb222ff704bbe29f0ab in 538ms, sequenceid=365, compaction requested=true 2024-12-03T05:16:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:40,714 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:40,714 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:40,715 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:40,715 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:40,715 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:40,715 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9baaa0fee1f34cd3a1c51f5440919388, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=36.7 K 2024-12-03T05:16:40,715 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:40,715 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:40,715 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:40,715 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=126.2 K 2024-12-03T05:16:40,715 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:40,715 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb] 2024-12-03T05:16:40,716 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920ac2bf77d04ecd9db8d2fa815d9c47, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995486 2024-12-03T05:16:40,716 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 9baaa0fee1f34cd3a1c51f5440919388, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995486 2024-12-03T05:16:40,716 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9142c8ddf22b45febe16aadff726bc99, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733202997829 2024-12-03T05:16:40,716 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c074fbe86297476a948572454d49df96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733202997829 2024-12-03T05:16:40,717 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f37baab9b58e450c8351c7ac01f17d8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000166 2024-12-03T05:16:40,717 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1b700214424e4da5f25f375831eafb, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000124 2024-12-03T05:16:40,725 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:40,725 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/1ccf6668018848f0ac1523901c133bdc is 50, key is test_row_0/B:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,728 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:40,742 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f4596b0558e64b4880a1ae53ab0fe473_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:40,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f4596b0558e64b4880a1ae53ab0fe473_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:40,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f4596b0558e64b4880a1ae53ab0fe473_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:40,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742266_1442 (size=13051) 2024-12-03T05:16:40,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742267_1443 (size=4469) 2024-12-03T05:16:40,766 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#367 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:40,766 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/eb1ff2d9ab2c402b9b6ae8f6757ee576 is 175, key is test_row_0/A:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,769 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/1ccf6668018848f0ac1523901c133bdc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/1ccf6668018848f0ac1523901c133bdc 2024-12-03T05:16:40,775 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into 1ccf6668018848f0ac1523901c133bdc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:40,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:40,775 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=13, startTime=1733203000714; duration=0sec 2024-12-03T05:16:40,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:40,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:40,775 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:16:40,776 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:16:40,776 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:40,776 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:40,776 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/fa7356acd96f459e939c450c8c1087b9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=36.7 K 2024-12-03T05:16:40,776 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7356acd96f459e939c450c8c1087b9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733202995486 2024-12-03T05:16:40,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3561e682c286422fb10e38a687d6dc67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733202997829 2024-12-03T05:16:40,777 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 223ef4a2234247628071b8fc9e20eec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000166 2024-12-03T05:16:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742268_1444 (size=32005) 2024-12-03T05:16:40,804 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:40,805 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/98c570c39a3c4fcbbbb96dbb77486d91 is 50, key is test_row_0/C:col10/1733203000166/Put/seqid=0 2024-12-03T05:16:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742269_1445 (size=13051) 2024-12-03T05:16:40,844 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/98c570c39a3c4fcbbbb96dbb77486d91 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/98c570c39a3c4fcbbbb96dbb77486d91 2024-12-03T05:16:40,854 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 98c570c39a3c4fcbbbb96dbb77486d91(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:40,854 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:40,854 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=13, startTime=1733203000714; duration=0sec 2024-12-03T05:16:40,854 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:40,854 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:41,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:16:41,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d8ecc4dfd77149e5807ddffc427c3752_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733203001012/Put/seqid=0 2024-12-03T05:16:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742270_1446 (size=14994) 2024-12-03T05:16:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203061052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203061052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203061170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203061170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,210 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/eb1ff2d9ab2c402b9b6ae8f6757ee576 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576 2024-12-03T05:16:41,214 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into eb1ff2d9ab2c402b9b6ae8f6757ee576(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:41,214 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:41,214 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=13, startTime=1733203000714; duration=0sec 2024-12-03T05:16:41,214 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:41,214 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203061390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203061390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,456 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:41,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d8ecc4dfd77149e5807ddffc427c3752_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8ecc4dfd77149e5807ddffc427c3752_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:41,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/890f898b8267454e8073d6b965a08897, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:41,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/890f898b8267454e8073d6b965a08897 is 175, key is test_row_0/A:col10/1733203001012/Put/seqid=0 2024-12-03T05:16:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742271_1447 (size=39949) 2024-12-03T05:16:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203061710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203061710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:41,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:41,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T05:16:41,851 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:41,851 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-03T05:16:41,854 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:41,855 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:41,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:41,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/890f898b8267454e8073d6b965a08897 2024-12-03T05:16:41,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/07d0a22ea5744ffa87fbfe7618364492 is 50, key is test_row_0/B:col10/1733203001012/Put/seqid=0 2024-12-03T05:16:41,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742272_1448 (size=12301) 2024-12-03T05:16:41,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/07d0a22ea5744ffa87fbfe7618364492 2024-12-03T05:16:41,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/6952cf5de28949f3a93ca9065a80a868 is 50, key is test_row_0/C:col10/1733203001012/Put/seqid=0 2024-12-03T05:16:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:41,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742273_1449 (size=12301) 2024-12-03T05:16:41,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/6952cf5de28949f3a93ca9065a80a868 2024-12-03T05:16:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/890f898b8267454e8073d6b965a08897 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897 2024-12-03T05:16:42,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897, entries=200, sequenceid=405, filesize=39.0 K 2024-12-03T05:16:42,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/07d0a22ea5744ffa87fbfe7618364492 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492 2024-12-03T05:16:42,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:42,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:42,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T05:16:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/6952cf5de28949f3a93ca9065a80a868 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868 2024-12-03T05:16:42,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T05:16:42,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8a02ea06b7843bb222ff704bbe29f0ab in 1011ms, sequenceid=405, compaction requested=false 2024-12-03T05:16:42,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:42,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:42,166 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032674746ec0924d20a72cc5fdf354977b_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733203001047/Put/seqid=0 2024-12-03T05:16:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742274_1450 (size=12454) 2024-12-03T05:16:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:42,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. as already flushing 2024-12-03T05:16:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203002302, max timestamp=1733203002312Num uniques 859; 2024-12-03T05:16:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203002302, max timestamp=1733203002312Num uniques 859; 2024-12-03T05:16:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203062369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203062369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:42,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203062490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203062490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:42,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032674746ec0924d20a72cc5fdf354977b_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032674746ec0924d20a72cc5fdf354977b_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/49510040b313480f85567e6b0a756c56, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/49510040b313480f85567e6b0a756c56 is 175, key is test_row_0/A:col10/1733203001047/Put/seqid=0 2024-12-03T05:16:42,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742275_1451 (size=31255) 2024-12-03T05:16:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203062710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203062710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203063030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203063031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:43,036 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/49510040b313480f85567e6b0a756c56 2024-12-03T05:16:43,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e115715132ea475d8a0deaee60dcca25 is 50, key is test_row_0/B:col10/1733203001047/Put/seqid=0 2024-12-03T05:16:43,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742276_1452 (size=12301) 2024-12-03T05:16:43,112 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-12-03T05:16:43,112 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:43,112 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,112 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:43,118 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-12-03T05:16:43,118 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:43,118 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,118 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:43,119 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-12-03T05:16:43,119 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:43,119 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,119 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:43,121 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-12-03T05:16:43,121 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:43,121 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,122 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:43,128 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-12-03T05:16:43,128 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:43,128 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,128 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:43,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:43,448 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e115715132ea475d8a0deaee60dcca25 2024-12-03T05:16:43,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/92f3a8c29c9c41bea15544daf1bc9e93 is 50, key is test_row_0/C:col10/1733203001047/Put/seqid=0 2024-12-03T05:16:43,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742277_1453 (size=12301) 2024-12-03T05:16:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53590 deadline: 1733203063541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1733203063541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:16:43,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/92f3a8c29c9c41bea15544daf1bc9e93 2024-12-03T05:16:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/49510040b313480f85567e6b0a756c56 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56 2024-12-03T05:16:43,869 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56, entries=150, sequenceid=415, filesize=30.5 K 2024-12-03T05:16:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/e115715132ea475d8a0deaee60dcca25 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25 2024-12-03T05:16:43,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25, entries=150, sequenceid=415, filesize=12.0 K 2024-12-03T05:16:43,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/92f3a8c29c9c41bea15544daf1bc9e93 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93 2024-12-03T05:16:43,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93, entries=150, sequenceid=415, filesize=12.0 K 2024-12-03T05:16:43,879 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 8a02ea06b7843bb222ff704bbe29f0ab in 1713ms, sequenceid=415, compaction requested=true 2024-12-03T05:16:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-03T05:16:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-03T05:16:43,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T05:16:43,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-12-03T05:16:43,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0320 sec 2024-12-03T05:16:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T05:16:44,001 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:44,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=145.36 KB heapSize=423.16 KB 2024-12-03T05:16:44,553 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-12-03T05:16:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:44,553 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:44,553 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:44,554 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:44,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:44,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:44,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:44,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c2dff72d44644bea77c3a8bfd77e8e3_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_0/A:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:44,562 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-12-03T05:16:44,562 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:44,562 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:44,562 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:44,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:44,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742278_1454 (size=12454) 2024-12-03T05:16:44,966 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:44,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c2dff72d44644bea77c3a8bfd77e8e3_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2dff72d44644bea77c3a8bfd77e8e3_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:44,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1b9d20b314d486f940e2f9fbb4bc7ed, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:44,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1b9d20b314d486f940e2f9fbb4bc7ed is 175, key is test_row_0/A:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742279_1455 (size=31255) 2024-12-03T05:16:45,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1b9d20b314d486f940e2f9fbb4bc7ed 2024-12-03T05:16:45,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f79321d3ae0348119c15e9c05804d308 is 50, key is test_row_0/B:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:45,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742280_1456 (size=12301) 2024-12-03T05:16:45,482 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:16:45,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f79321d3ae0348119c15e9c05804d308 2024-12-03T05:16:45,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/2ba309e8045f4d6696fdf17041c9eb07 is 50, key is test_row_0/C:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:45,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742281_1457 (size=12301) 2024-12-03T05:16:45,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/2ba309e8045f4d6696fdf17041c9eb07 2024-12-03T05:16:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/f1b9d20b314d486f940e2f9fbb4bc7ed as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed 2024-12-03T05:16:45,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed, entries=150, sequenceid=449, filesize=30.5 K 2024-12-03T05:16:45,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/f79321d3ae0348119c15e9c05804d308 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308 2024-12-03T05:16:45,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T05:16:45,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/2ba309e8045f4d6696fdf17041c9eb07 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07 2024-12-03T05:16:45,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T05:16:45,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~423.11 KB/433264, currentSize=6.71 KB/6870 for 8a02ea06b7843bb222ff704bbe29f0ab in 1262ms, sequenceid=449, compaction requested=true 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:45,815 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a02ea06b7843bb222ff704bbe29f0ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:45,815 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:45,816 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:45,816 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:45,816 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/A is initiating minor compaction (all files) 2024-12-03T05:16:45,816 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/B is initiating minor compaction (all files) 2024-12-03T05:16:45,816 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/B in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:45,816 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/A in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:45,816 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=131.3 K 2024-12-03T05:16:45,816 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/1ccf6668018848f0ac1523901c133bdc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=48.8 K 2024-12-03T05:16:45,816 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:45,816 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed] 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccf6668018848f0ac1523901c133bdc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000166 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1ff2d9ab2c402b9b6ae8f6757ee576, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000166 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d0a22ea5744ffa87fbfe7618364492, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733203000349 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890f898b8267454e8073d6b965a08897, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733203000336 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e115715132ea475d8a0deaee60dcca25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733203001034 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49510040b313480f85567e6b0a756c56, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733203001034 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b9d20b314d486f940e2f9fbb4bc7ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733203002349 2024-12-03T05:16:45,817 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f79321d3ae0348119c15e9c05804d308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733203002349 2024-12-03T05:16:45,831 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:45,831 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/95aa9d3a0eb94217940e5086a7d15cd7 is 50, key is test_row_0/B:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:45,837 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:45,849 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036b1c7044d941460cb97446ef0c6fdc22_8a02ea06b7843bb222ff704bbe29f0ab store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:45,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742282_1458 (size=13187) 2024-12-03T05:16:45,856 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/95aa9d3a0eb94217940e5086a7d15cd7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/95aa9d3a0eb94217940e5086a7d15cd7 2024-12-03T05:16:45,860 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/B of 8a02ea06b7843bb222ff704bbe29f0ab into 95aa9d3a0eb94217940e5086a7d15cd7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:45,860 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:45,861 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/B, priority=12, startTime=1733203005815; duration=0sec 2024-12-03T05:16:45,861 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:45,861 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:B 2024-12-03T05:16:45,861 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:45,862 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:45,862 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 8a02ea06b7843bb222ff704bbe29f0ab/C is initiating minor compaction (all files) 2024-12-03T05:16:45,862 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a02ea06b7843bb222ff704bbe29f0ab/C in TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:45,862 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/98c570c39a3c4fcbbbb96dbb77486d91, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp, totalSize=48.8 K 2024-12-03T05:16:45,862 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c570c39a3c4fcbbbb96dbb77486d91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733203000166 2024-12-03T05:16:45,863 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6952cf5de28949f3a93ca9065a80a868, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733203000349 2024-12-03T05:16:45,863 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f3a8c29c9c41bea15544daf1bc9e93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733203001034 2024-12-03T05:16:45,863 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba309e8045f4d6696fdf17041c9eb07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733203002349 2024-12-03T05:16:45,865 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036b1c7044d941460cb97446ef0c6fdc22_8a02ea06b7843bb222ff704bbe29f0ab, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:45,866 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036b1c7044d941460cb97446ef0c6fdc22_8a02ea06b7843bb222ff704bbe29f0ab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:45,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742283_1459 (size=4469) 2024-12-03T05:16:45,880 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#A#compaction#379 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:45,880 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/582fb069536f4ea587b4ef5bc4d887d7 is 175, key is test_row_0/A:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:45,891 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a02ea06b7843bb222ff704bbe29f0ab#C#compaction#380 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:45,891 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/9c3a5ee65dc94dd2aef809ab77a90abb is 50, key is test_row_0/C:col10/1733203004551/Put/seqid=0 2024-12-03T05:16:45,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742284_1460 (size=32141) 2024-12-03T05:16:45,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742285_1461 (size=13187) 2024-12-03T05:16:46,302 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/582fb069536f4ea587b4ef5bc4d887d7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/582fb069536f4ea587b4ef5bc4d887d7 2024-12-03T05:16:46,310 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/A of 8a02ea06b7843bb222ff704bbe29f0ab into 582fb069536f4ea587b4ef5bc4d887d7(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:46,310 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:46,310 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/A, priority=12, startTime=1733203005815; duration=0sec 2024-12-03T05:16:46,310 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:46,310 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:A 2024-12-03T05:16:46,322 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/9c3a5ee65dc94dd2aef809ab77a90abb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9c3a5ee65dc94dd2aef809ab77a90abb 2024-12-03T05:16:46,326 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a02ea06b7843bb222ff704bbe29f0ab/C of 8a02ea06b7843bb222ff704bbe29f0ab into 9c3a5ee65dc94dd2aef809ab77a90abb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:46,326 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a02ea06b7843bb222ff704bbe29f0ab: 2024-12-03T05:16:46,326 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., storeName=8a02ea06b7843bb222ff704bbe29f0ab/C, priority=12, startTime=1733203005815; duration=0sec 2024-12-03T05:16:46,326 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:46,326 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a02ea06b7843bb222ff704bbe29f0ab:C 2024-12-03T05:16:48,862 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-12-03T05:16:48,862 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:48,862 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,863 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:48,893 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-12-03T05:16:48,893 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:48,893 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,894 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:48,902 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-12-03T05:16:48,902 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:16:48,902 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,902 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 938 2024-12-03T05:16:48,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2801 rows 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 935 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2804 rows 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2871 rows 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2837 rows 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 945 2024-12-03T05:16:48,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2835 rows 2024-12-03T05:16:48,903 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:16:48,903 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:16:48,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:48,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:16:48,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:16:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:48,907 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203008906"}]},"ts":"1733203008906"} 2024-12-03T05:16:48,908 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:16:48,908 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:16:48,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:16:48,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, UNASSIGN}] 2024-12-03T05:16:48,910 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, UNASSIGN 2024-12-03T05:16:48,911 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:48,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, UNASSIGN because future has completed 2024-12-03T05:16:48,913 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:16:48,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:49,065 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 8a02ea06b7843bb222ff704bbe29f0ab, disabling compactions & flushes 2024-12-03T05:16:49,066 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. after waiting 0 ms 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:49,066 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 8a02ea06b7843bb222ff704bbe29f0ab 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=A 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=B 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a02ea06b7843bb222ff704bbe29f0ab, store=C 2024-12-03T05:16:49,066 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:49,071 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034211bfa632344975a44570d400d39858_8a02ea06b7843bb222ff704bbe29f0ab is 50, key is test_row_1/A:col10/1733203008901/Put/seqid=0 2024-12-03T05:16:49,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742286_1462 (size=9914) 2024-12-03T05:16:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:49,474 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:49,479 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034211bfa632344975a44570d400d39858_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034211bfa632344975a44570d400d39858_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:49,480 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/24b7280e1ba6467ea1202b4ee839b6db, store: [table=TestAcidGuarantees family=A region=8a02ea06b7843bb222ff704bbe29f0ab] 2024-12-03T05:16:49,481 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/24b7280e1ba6467ea1202b4ee839b6db is 175, key is test_row_1/A:col10/1733203008901/Put/seqid=0 2024-12-03T05:16:49,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742287_1463 (size=22561) 2024-12-03T05:16:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:49,885 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/24b7280e1ba6467ea1202b4ee839b6db 2024-12-03T05:16:49,891 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c3c49724f0d649e0bd062de20ad2a363 is 50, key is test_row_1/B:col10/1733203008901/Put/seqid=0 2024-12-03T05:16:49,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742288_1464 (size=9857) 2024-12-03T05:16:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:50,085 WARN [regionserver/e3c6570475b6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-03T05:16:50,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed] to archive 2024-12-03T05:16:50,087 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:50,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3bf4137a30794f2499799771d85875b6 2024-12-03T05:16:50,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c48e22c7d88c4bf5b0e57c48038e64f9 2024-12-03T05:16:50,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/fb876b87209c4b9eadce709e6a5bf7d3 2024-12-03T05:16:50,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/b187e25769dd4391bb9307a576d908c7 2024-12-03T05:16:50,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/0909e19ba5cf44e884736ba35bf1e190 2024-12-03T05:16:50,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c18f57835d1646068a25e4fd234c7943 2024-12-03T05:16:50,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c5bfc7f79ce84401a8cf8f039cd22fcd 2024-12-03T05:16:50,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/3818eaf10292444ba17e93d87ea7721e 2024-12-03T05:16:50,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/7f98b35bf5a54d8f87de5bcbf46f23e5 2024-12-03T05:16:50,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/8325a7fa3e5b4233a51f0caf65017683 2024-12-03T05:16:50,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ad5d88b9ead4470a9ac667fadd780682 2024-12-03T05:16:50,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1424f91e23b4b739f5dab1d6ba68412 2024-12-03T05:16:50,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/82d96843851f4d33a688d0a5fb487a82 2024-12-03T05:16:50,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9b0b8eaad92f4884b0e92d4597ba4b5d 2024-12-03T05:16:50,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/4fed620375fb4803b74d8933996c5944 2024-12-03T05:16:50,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/437f003cb03147a789503c49157e8e51 2024-12-03T05:16:50,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/da735daee7424b15b68d8c3e1c48586b 2024-12-03T05:16:50,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/c8cd35ce323e4b4da788a6446ab63828 2024-12-03T05:16:50,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/69910c3273f748c59e9b50653953eabd 2024-12-03T05:16:50,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/920ac2bf77d04ecd9db8d2fa815d9c47 2024-12-03T05:16:50,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/9142c8ddf22b45febe16aadff726bc99 2024-12-03T05:16:50,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/ed1b700214424e4da5f25f375831eafb 2024-12-03T05:16:50,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/eb1ff2d9ab2c402b9b6ae8f6757ee576 2024-12-03T05:16:50,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/890f898b8267454e8073d6b965a08897 2024-12-03T05:16:50,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/49510040b313480f85567e6b0a756c56 2024-12-03T05:16:50,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/f1b9d20b314d486f940e2f9fbb4bc7ed 2024-12-03T05:16:50,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f736b2b9e600466988ae971dbe7156d8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e4dfa9ab86434210986729584eb99299, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3e9906424b194681b7087c31e45b72ad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/941c7047cb1f4214acbd40d04e06bafd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/a35cb0465bd644c7b1a552e57dc4c364, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9baaa0fee1f34cd3a1c51f5440919388, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/1ccf6668018848f0ac1523901c133bdc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308] to archive 2024-12-03T05:16:50,176 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:50,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/87b8ba4192984d26aab81ddf400a1ba0 2024-12-03T05:16:50,178 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/527efb34c6c349d2ad134c56d519ea05 2024-12-03T05:16:50,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f736b2b9e600466988ae971dbe7156d8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f736b2b9e600466988ae971dbe7156d8 2024-12-03T05:16:50,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e486843548c5417b9ee87977cd89bab9 2024-12-03T05:16:50,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/224dcf01808948849a2c4bfd605b3762 2024-12-03T05:16:50,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e4dfa9ab86434210986729584eb99299 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e4dfa9ab86434210986729584eb99299 2024-12-03T05:16:50,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/7d4d9d81c3f942178ba52ab30ba492d6 2024-12-03T05:16:50,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e48d042bdd894162900c96f1585ed405 2024-12-03T05:16:50,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3e9906424b194681b7087c31e45b72ad to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3e9906424b194681b7087c31e45b72ad 2024-12-03T05:16:50,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/ea9f727d4b0f422c82791f536e122b18 2024-12-03T05:16:50,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e433bdf1a22d4f2d886849c5ac58f102 2024-12-03T05:16:50,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/941c7047cb1f4214acbd40d04e06bafd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/941c7047cb1f4214acbd40d04e06bafd 2024-12-03T05:16:50,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/38a5d75575c34006b39032e4fb650ec5 2024-12-03T05:16:50,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/5943c52a92f849e1a87651728542a11f 2024-12-03T05:16:50,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/3717be82560a4c6cb0abed178dfdbdcf 2024-12-03T05:16:50,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/a35cb0465bd644c7b1a552e57dc4c364 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/a35cb0465bd644c7b1a552e57dc4c364 2024-12-03T05:16:50,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/228e8627dce243afa7e088b4dab29aed 2024-12-03T05:16:50,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/028731f950ec469ebc624a4141b7fc71 2024-12-03T05:16:50,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9baaa0fee1f34cd3a1c51f5440919388 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9baaa0fee1f34cd3a1c51f5440919388 2024-12-03T05:16:50,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/9734356bc9064aea9528982ff3f2be7e 2024-12-03T05:16:50,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c074fbe86297476a948572454d49df96 2024-12-03T05:16:50,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/1ccf6668018848f0ac1523901c133bdc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/1ccf6668018848f0ac1523901c133bdc 2024-12-03T05:16:50,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f37baab9b58e450c8351c7ac01f17d8b 2024-12-03T05:16:50,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/07d0a22ea5744ffa87fbfe7618364492 2024-12-03T05:16:50,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/e115715132ea475d8a0deaee60dcca25 2024-12-03T05:16:50,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/f79321d3ae0348119c15e9c05804d308 2024-12-03T05:16:50,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/5cb123c3b58a4a5483e4aa0670a009fd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3a1916c948ae4480ae3997d5ae5b272b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/a54eb94c59dd4f00a4ef2786a260ab83, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6b3add75404944609b561b76fcd92255, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0bfbfa1f9cac4f2e9dc5b83c4062109e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/fa7356acd96f459e939c450c8c1087b9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/98c570c39a3c4fcbbbb96dbb77486d91, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07] to archive 2024-12-03T05:16:50,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:16:50,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/c32ad39400114af38b4b793544179b5a 2024-12-03T05:16:50,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/bade5b43b38544c4b22dc06df410b06e 2024-12-03T05:16:50,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/5cb123c3b58a4a5483e4aa0670a009fd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/5cb123c3b58a4a5483e4aa0670a009fd 2024-12-03T05:16:50,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/7e9a8d57134a44da872e3d92894e543e 2024-12-03T05:16:50,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/451aa0aaddf44dc3a26a95653c8a1616 2024-12-03T05:16:50,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3a1916c948ae4480ae3997d5ae5b272b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3a1916c948ae4480ae3997d5ae5b272b 2024-12-03T05:16:50,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d6b325b3eac04abca23cff7aaad130ea 2024-12-03T05:16:50,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3951881c618849f6b715fe5088e7ebc1 2024-12-03T05:16:50,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/a54eb94c59dd4f00a4ef2786a260ab83 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/a54eb94c59dd4f00a4ef2786a260ab83 2024-12-03T05:16:50,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/185110bb45e24c66a0ed631d9a035671 2024-12-03T05:16:50,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0f57fcfceb0c4212889b8fda0bf7a949 2024-12-03T05:16:50,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6b3add75404944609b561b76fcd92255 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6b3add75404944609b561b76fcd92255 2024-12-03T05:16:50,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/422f5c81faad438aab88dd1724ae91d3 2024-12-03T05:16:50,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9e4efe9ff97b439d92e68899d1685286 2024-12-03T05:16:50,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/aeae67b09a0d4cb29eefae88578a8622 2024-12-03T05:16:50,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0bfbfa1f9cac4f2e9dc5b83c4062109e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/0bfbfa1f9cac4f2e9dc5b83c4062109e 2024-12-03T05:16:50,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/63e832e0ca894af2b909aeae36e0d6c5 2024-12-03T05:16:50,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/394f3edecd68436191a07175b8cb185f 2024-12-03T05:16:50,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/fa7356acd96f459e939c450c8c1087b9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/fa7356acd96f459e939c450c8c1087b9 2024-12-03T05:16:50,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/d7177737550a4e0599b3ac18955f388e 2024-12-03T05:16:50,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/3561e682c286422fb10e38a687d6dc67 2024-12-03T05:16:50,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/98c570c39a3c4fcbbbb96dbb77486d91 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/98c570c39a3c4fcbbbb96dbb77486d91 2024-12-03T05:16:50,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/223ef4a2234247628071b8fc9e20eec6 2024-12-03T05:16:50,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/6952cf5de28949f3a93ca9065a80a868 2024-12-03T05:16:50,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/92f3a8c29c9c41bea15544daf1bc9e93 2024-12-03T05:16:50,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/2ba309e8045f4d6696fdf17041c9eb07 2024-12-03T05:16:50,298 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c3c49724f0d649e0bd062de20ad2a363 2024-12-03T05:16:50,305 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/319f5e7353bd47ab89ca16f01fa52ac5 is 50, key is test_row_1/C:col10/1733203008901/Put/seqid=0 2024-12-03T05:16:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742289_1465 (size=9857) 2024-12-03T05:16:50,710 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/319f5e7353bd47ab89ca16f01fa52ac5 2024-12-03T05:16:50,714 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/A/24b7280e1ba6467ea1202b4ee839b6db as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/24b7280e1ba6467ea1202b4ee839b6db 2024-12-03T05:16:50,716 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/24b7280e1ba6467ea1202b4ee839b6db, entries=100, sequenceid=459, filesize=22.0 K 2024-12-03T05:16:50,717 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/B/c3c49724f0d649e0bd062de20ad2a363 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c3c49724f0d649e0bd062de20ad2a363 2024-12-03T05:16:50,719 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c3c49724f0d649e0bd062de20ad2a363, entries=100, sequenceid=459, filesize=9.6 K 2024-12-03T05:16:50,720 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/.tmp/C/319f5e7353bd47ab89ca16f01fa52ac5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/319f5e7353bd47ab89ca16f01fa52ac5 2024-12-03T05:16:50,723 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/319f5e7353bd47ab89ca16f01fa52ac5, entries=100, sequenceid=459, filesize=9.6 K 2024-12-03T05:16:50,724 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8a02ea06b7843bb222ff704bbe29f0ab in 1657ms, sequenceid=459, compaction requested=false 2024-12-03T05:16:50,728 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-12-03T05:16:50,729 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. 2024-12-03T05:16:50,729 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 8a02ea06b7843bb222ff704bbe29f0ab: Waiting for close lock at 1733203009066Running coprocessor pre-close hooks at 1733203009066Disabling compacts and flushes for region at 1733203009066Disabling writes for close at 1733203009066Obtaining lock to block concurrent updates at 1733203009066Preparing flush snapshotting stores in 8a02ea06b7843bb222ff704bbe29f0ab at 1733203009066Finished memstore snapshotting TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733203009066Flushing stores of TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab. at 1733203009067 (+1 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/A: creating writer at 1733203009067Flushing mob file 8a02ea06b7843bb222ff704bbe29f0ab/A: appending metadata at 1733203009070 (+3 ms)Flushing mob file 8a02ea06b7843bb222ff704bbe29f0ab/A: closing flushed file at 1733203009070Flushing 8a02ea06b7843bb222ff704bbe29f0ab/A: appending metadata at 1733203009480 (+410 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/A: closing flushed file at 1733203009480Flushing 8a02ea06b7843bb222ff704bbe29f0ab/B: creating writer at 1733203009889 (+409 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/B: appending metadata at 1733203009891 (+2 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/B: closing flushed file at 1733203009891Flushing 8a02ea06b7843bb222ff704bbe29f0ab/C: creating writer at 1733203010301 (+410 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/C: appending metadata at 1733203010304 (+3 ms)Flushing 8a02ea06b7843bb222ff704bbe29f0ab/C: closing flushed file at 1733203010304Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dc3281c: reopening flushed file at 1733203010713 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@307ba1cc: reopening flushed file at 1733203010716 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23883faa: reopening flushed file at 1733203010719 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8a02ea06b7843bb222ff704bbe29f0ab in 1657ms, sequenceid=459, compaction requested=false at 1733203010724 (+5 ms)Writing region close event to WAL at 1733203010725 (+1 ms)Running coprocessor post-close hooks at 1733203010729 (+4 ms)Closed at 1733203010729 2024-12-03T05:16:50,731 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:50,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=8a02ea06b7843bb222ff704bbe29f0ab, regionState=CLOSED 2024-12-03T05:16:50,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:50,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T05:16:50,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a02ea06b7843bb222ff704bbe29f0ab, server=e3c6570475b6,37863,1733202887099 in 1.8200 sec 2024-12-03T05:16:50,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-03T05:16:50,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a02ea06b7843bb222ff704bbe29f0ab, UNASSIGN in 1.8250 sec 2024-12-03T05:16:50,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T05:16:50,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8290 sec 2024-12-03T05:16:50,739 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203010739"}]},"ts":"1733203010739"} 2024-12-03T05:16:50,740 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:16:50,740 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:16:50,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8360 sec 2024-12-03T05:16:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T05:16:51,051 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:51,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:16:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,053 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T05:16:51,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,056 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,058 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits] 2024-12-03T05:16:51,061 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/24b7280e1ba6467ea1202b4ee839b6db to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/24b7280e1ba6467ea1202b4ee839b6db 2024-12-03T05:16:51,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/582fb069536f4ea587b4ef5bc4d887d7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/A/582fb069536f4ea587b4ef5bc4d887d7 2024-12-03T05:16:51,066 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/95aa9d3a0eb94217940e5086a7d15cd7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/95aa9d3a0eb94217940e5086a7d15cd7 2024-12-03T05:16:51,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c3c49724f0d649e0bd062de20ad2a363 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/B/c3c49724f0d649e0bd062de20ad2a363 2024-12-03T05:16:51,071 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/319f5e7353bd47ab89ca16f01fa52ac5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/319f5e7353bd47ab89ca16f01fa52ac5 2024-12-03T05:16:51,072 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9c3a5ee65dc94dd2aef809ab77a90abb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/C/9c3a5ee65dc94dd2aef809ab77a90abb 2024-12-03T05:16:51,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits/462.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab/recovered.edits/462.seqid 2024-12-03T05:16:51,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,076 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:16:51,076 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:16:51,077 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T05:16:51,081 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203030b73c72cb84bbc84a77a52834a1c4c_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203030b73c72cb84bbc84a77a52834a1c4c_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,082 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031452ae54034e4d64af27fb7dfb930b1d_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031452ae54034e4d64af27fb7dfb930b1d_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,084 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314ff2eb6e9674c51a3f60082c2368cdd_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120314ff2eb6e9674c51a3f60082c2368cdd_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,085 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031b56e334f6374d18b7abc4514523cbf4_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031b56e334f6374d18b7abc4514523cbf4_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,086 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e8ae975f4e149e39c2842252228f62f_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e8ae975f4e149e39c2842252228f62f_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,087 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032674746ec0924d20a72cc5fdf354977b_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032674746ec0924d20a72cc5fdf354977b_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,089 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034211bfa632344975a44570d400d39858_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034211bfa632344975a44570d400d39858_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,090 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d635712219e418c9cd4b34ccad9c99d_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d635712219e418c9cd4b34ccad9c99d_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,091 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2dff72d44644bea77c3a8bfd77e8e3_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2dff72d44644bea77c3a8bfd77e8e3_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,092 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2e363a740442afaeb39a73993cc379_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c2e363a740442afaeb39a73993cc379_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,093 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ef149cf8c824bc792b9ce21c2f6e2d0_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037ef149cf8c824bc792b9ce21c2f6e2d0_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,094 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393cafe83c453485a96077160dd13614e_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393cafe83c453485a96077160dd13614e_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,095 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b15d8164726b41b2a31361ebc1d903a1_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b15d8164726b41b2a31361ebc1d903a1_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,096 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2a2cdbea55542eaa11a46ff75f75331_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2a2cdbea55542eaa11a46ff75f75331_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,097 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caf381510e49407782ea72dfcaeb1478_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caf381510e49407782ea72dfcaeb1478_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,098 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d15d3a4b4f264389926deba4158f7d3a_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d15d3a4b4f264389926deba4158f7d3a_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,100 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d258237c81484e53b3685391372716fb_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d258237c81484e53b3685391372716fb_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,101 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8ecc4dfd77149e5807ddffc427c3752_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8ecc4dfd77149e5807ddffc427c3752_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,103 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e49526978e144ab18917a305149bd9ae_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e49526978e144ab18917a305149bd9ae_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,104 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff4a3c4504404c5da5b58782ebe67ef0_8a02ea06b7843bb222ff704bbe29f0ab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff4a3c4504404c5da5b58782ebe67ef0_8a02ea06b7843bb222ff704bbe29f0ab 2024-12-03T05:16:51,105 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:16:51,107 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,109 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:16:51,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:16:51,112 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,112 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:16:51,112 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733203011112"}]},"ts":"9223372036854775807"} 2024-12-03T05:16:51,114 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:16:51,114 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8a02ea06b7843bb222ff704bbe29f0ab, NAME => 'TestAcidGuarantees,,1733202979886.8a02ea06b7843bb222ff704bbe29f0ab.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:16:51,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:16:51,114 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733203011114"}]},"ts":"9223372036854775807"} 2024-12-03T05:16:51,116 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:16:51,117 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-12-03T05:16:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T05:16:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:16:51,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:51,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=757 (was 774), ProcessCount=11 (was 11), AvailableMemoryMB=3021 (was 2440) - AvailableMemoryMB LEAK? - 2024-12-03T05:16:51,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=757, ProcessCount=11, AvailableMemoryMB=3021 2024-12-03T05:16:51,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:16:51,200 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:16:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:16:51,203 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:16:51,203 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:51,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-03T05:16:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:51,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:16:51,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742290_1466 (size=1061) 2024-12-03T05:16:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:51,611 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:16:51,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742291_1467 (size=53) 2024-12-03T05:16:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6596af7287e3b5c33fe33f24d375a15b, disabling compactions & flushes 2024-12-03T05:16:52,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. after waiting 0 ms 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6596af7287e3b5c33fe33f24d375a15b: Waiting for close lock at 1733203012130Disabling compacts and flushes for region at 1733203012130Disabling writes for close at 1733203012130Writing region close event to WAL at 1733203012130Closed at 1733203012130 2024-12-03T05:16:52,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:16:52,132 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733203012132"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733203012132"}]},"ts":"1733203012132"} 2024-12-03T05:16:52,141 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:16:52,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:16:52,147 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203012147"}]},"ts":"1733203012147"} 2024-12-03T05:16:52,152 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:16:52,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, ASSIGN}] 2024-12-03T05:16:52,154 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, ASSIGN 2024-12-03T05:16:52,156 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:16:52,306 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6596af7287e3b5c33fe33f24d375a15b, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:52,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, ASSIGN because future has completed 2024-12-03T05:16:52,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:52,464 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,464 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:16:52,465 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,465 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:16:52,465 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,465 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,467 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,468 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:52,468 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6596af7287e3b5c33fe33f24d375a15b columnFamilyName A 2024-12-03T05:16:52,468 DEBUG [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:52,469 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(327): Store=6596af7287e3b5c33fe33f24d375a15b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:52,469 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,470 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:52,470 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6596af7287e3b5c33fe33f24d375a15b columnFamilyName B 2024-12-03T05:16:52,470 DEBUG [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:52,471 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(327): Store=6596af7287e3b5c33fe33f24d375a15b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:52,471 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,472 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:16:52,472 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6596af7287e3b5c33fe33f24d375a15b columnFamilyName C 2024-12-03T05:16:52,472 DEBUG [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:16:52,472 INFO [StoreOpener-6596af7287e3b5c33fe33f24d375a15b-1 {}] regionserver.HStore(327): Store=6596af7287e3b5c33fe33f24d375a15b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:16:52,473 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,473 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,475 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,475 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,476 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,476 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,476 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:16:52,477 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,479 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:16:52,479 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 6596af7287e3b5c33fe33f24d375a15b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60059887, jitterRate=-0.10503794252872467}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:16:52,479 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:52,480 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 6596af7287e3b5c33fe33f24d375a15b: Running coprocessor pre-open hook at 1733203012465Writing region info on filesystem at 1733203012465Initializing all the Stores at 1733203012466 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203012467 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203012467Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203012467Cleaning up temporary data from old regions at 1733203012476 (+9 ms)Running coprocessor post-open hooks at 1733203012479 (+3 ms)Region opened successfully at 1733203012480 (+1 ms) 2024-12-03T05:16:52,480 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., pid=116, masterSystemTime=1733203012461 2024-12-03T05:16:52,482 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,482 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:52,482 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6596af7287e3b5c33fe33f24d375a15b, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:52,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:16:52,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T05:16:52,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 in 175 msec 2024-12-03T05:16:52,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-03T05:16:52,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, ASSIGN in 333 msec 2024-12-03T05:16:52,488 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:16:52,488 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203012488"}]},"ts":"1733203012488"} 2024-12-03T05:16:52,489 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:16:52,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:16:52,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2900 sec 2024-12-03T05:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T05:16:53,351 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:53,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3a0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,353 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d93bf9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,355 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d52741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,356 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:16:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:16:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c17fd0a 2024-12-03T05:16:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:16:53,361 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:16:53,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c2b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,363 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31927899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21472253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57af2b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfefd81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,367 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c83d4ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ae5a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,369 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b2575b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a0309c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fe770e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,373 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f994a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9afe04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cc712e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435a769a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755041e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,379 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318b43fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,380 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4a12b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b7256f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,382 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0b2176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d950c36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188c1103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,385 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14277c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,386 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421186d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f46cad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,388 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76140a54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eae5e76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d22d375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:16:53,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:16:53,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:16:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:16:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf60bf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:16:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:16:53,393 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:16:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561c6480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:16:53,394 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,394 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,394 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,395 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,396 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,397 DEBUG [Thread-2104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,397 DEBUG [Thread-2102 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:53,398 DEBUG [Thread-2102 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T05:16:53,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,399 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:53,400 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:53,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:53,400 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,401 DEBUG [Thread-2098 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,401 DEBUG [Thread-2098 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,403 DEBUG [Thread-2093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,403 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,405 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:16:53,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:16:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:16:53,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:16:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:53,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2] 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:53,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/fbad58e7916c460ba66f742a15e91b19 is 50, key is test_row_0/A:col10/1733203013409/Put/seqid=0 2024-12-03T05:16:53,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203013434, max timestamp=1733203013446Num uniques 859; 2024-12-03T05:16:53,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203013434, max timestamp=1733203013446Num uniques 859; 2024-12-03T05:16:53,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:53,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:53,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203013434, max timestamp=1733203013446Num uniques 859; 2024-12-03T05:16:53,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:16:53,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742292_1468 (size=12001) 2024-12-03T05:16:53,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203073488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203073489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203073491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203073492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203073496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:53,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203073601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203073601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203073611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203073612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203073612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:53,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203073810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203073811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203073820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203073820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203073820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:53,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/fbad58e7916c460ba66f742a15e91b19 2024-12-03T05:16:53,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:53,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:53,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/329a580cdf6f460fbf383b2a7b42e513 is 50, key is test_row_0/B:col10/1733203013409/Put/seqid=0 2024-12-03T05:16:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742293_1469 (size=12001) 2024-12-03T05:16:53,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/329a580cdf6f460fbf383b2a7b42e513 2024-12-03T05:16:53,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/87c9fb49d103489d876e1737c2cc5556 is 50, key is test_row_0/C:col10/1733203013409/Put/seqid=0 2024-12-03T05:16:53,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742294_1470 (size=12001) 2024-12-03T05:16:53,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/87c9fb49d103489d876e1737c2cc5556 2024-12-03T05:16:54,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/fbad58e7916c460ba66f742a15e91b19 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19 2024-12-03T05:16:54,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/329a580cdf6f460fbf383b2a7b42e513 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513 2024-12-03T05:16:54,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:16:54,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/87c9fb49d103489d876e1737c2cc5556 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556 2024-12-03T05:16:54,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T05:16:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:54,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6596af7287e3b5c33fe33f24d375a15b in 618ms, sequenceid=12, compaction requested=false 2024-12-03T05:16:54,031 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T05:16:54,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:54,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:54,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/d474537d32f147c1a6967e969d41c27f is 50, key is test_row_0/A:col10/1733203013484/Put/seqid=0 2024-12-03T05:16:54,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742295_1471 (size=14341) 2024-12-03T05:16:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203074135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203074136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203074138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203074140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203074144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203074251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203074251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203074251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203074260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203074261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203074460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203074460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203074460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203074481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203074492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/d474537d32f147c1a6967e969d41c27f 2024-12-03T05:16:54,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/bb5fe59da94c45f48c19efe0984adec5 is 50, key is test_row_0/B:col10/1733203013484/Put/seqid=0 2024-12-03T05:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:54,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742296_1472 (size=12001) 2024-12-03T05:16:54,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203074770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203074770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203074771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203074801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203074801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:54,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:54,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:54,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/bb5fe59da94c45f48c19efe0984adec5 2024-12-03T05:16:54,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d7a7efc1fbd44776b513f4d0ae2cccb3 is 50, key is test_row_0/C:col10/1733203013484/Put/seqid=0 2024-12-03T05:16:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742297_1473 (size=12001) 2024-12-03T05:16:55,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d7a7efc1fbd44776b513f4d0ae2cccb3 2024-12-03T05:16:55,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/d474537d32f147c1a6967e969d41c27f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f 2024-12-03T05:16:55,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f, entries=200, sequenceid=49, filesize=14.0 K 2024-12-03T05:16:55,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/bb5fe59da94c45f48c19efe0984adec5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5 2024-12-03T05:16:55,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T05:16:55,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d7a7efc1fbd44776b513f4d0ae2cccb3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3 2024-12-03T05:16:55,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T05:16:55,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6596af7287e3b5c33fe33f24d375a15b in 952ms, sequenceid=49, compaction requested=false 2024-12-03T05:16:55,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:55,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:55,089 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:55,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:55,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5825f809fa0e4991a932c2fe48478465 is 50, key is test_row_0/A:col10/1733203014135/Put/seqid=0 2024-12-03T05:16:55,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742298_1474 (size=12001) 2024-12-03T05:16:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:55,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:55,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203015334, max timestamp=1733203015385Num uniques 859; 2024-12-03T05:16:55,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:55,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203015334, max timestamp=1733203015385Num uniques 859; 2024-12-03T05:16:55,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203015334, max timestamp=1733203015385Num uniques 859; 2024-12-03T05:16:55,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:55,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:16:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203075441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203075453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203075473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203075481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203075488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:55,572 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5825f809fa0e4991a932c2fe48478465 2024-12-03T05:16:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/26b53300b5084ba983012ce0cca2b26a is 50, key is test_row_0/B:col10/1733203014135/Put/seqid=0 2024-12-03T05:16:55,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203075599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742299_1475 (size=12001) 2024-12-03T05:16:55,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203075607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203075607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203075617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203075620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203075832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203075845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203075845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203075845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203075853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/26b53300b5084ba983012ce0cca2b26a 2024-12-03T05:16:56,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5490d01057f342d09c5599af577d5016 is 50, key is test_row_0/C:col10/1733203014135/Put/seqid=0 2024-12-03T05:16:56,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742300_1476 (size=12001) 2024-12-03T05:16:56,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5490d01057f342d09c5599af577d5016 2024-12-03T05:16:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5825f809fa0e4991a932c2fe48478465 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465 2024-12-03T05:16:56,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T05:16:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/26b53300b5084ba983012ce0cca2b26a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a 2024-12-03T05:16:56,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203076173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203076181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203076181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203076181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203076184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T05:16:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5490d01057f342d09c5599af577d5016 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016 2024-12-03T05:16:56,211 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T05:16:56,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6596af7287e3b5c33fe33f24d375a15b in 1124ms, sequenceid=59, compaction requested=true 2024-12-03T05:16:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T05:16:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T05:16:56,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T05:16:56,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8250 sec 2024-12-03T05:16:56,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.8290 sec 2024-12-03T05:16:56,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:16:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:56,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:56,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:56,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a22c02e098804ee985a2a9da61a8a979 is 50, key is test_row_0/A:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203076736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742301_1477 (size=14341) 2024-12-03T05:16:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a22c02e098804ee985a2a9da61a8a979 2024-12-03T05:16:56,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203076744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203076745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203076748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203076749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/5839b0ae8ab744eda62a887ffe9dc95c is 50, key is test_row_0/B:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:56,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742302_1478 (size=12001) 2024-12-03T05:16:56,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203076860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203076871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203076871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203076873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203076873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203077072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203077104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203077105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203077105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203077105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/5839b0ae8ab744eda62a887ffe9dc95c 2024-12-03T05:16:57,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6c42aa2b936b4a2c86d1440a2f51446a is 50, key is test_row_0/C:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:57,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742303_1479 (size=12001) 2024-12-03T05:16:57,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6c42aa2b936b4a2c86d1440a2f51446a 2024-12-03T05:16:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a22c02e098804ee985a2a9da61a8a979 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979 2024-12-03T05:16:57,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979, entries=200, sequenceid=96, filesize=14.0 K 2024-12-03T05:16:57,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/5839b0ae8ab744eda62a887ffe9dc95c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c 2024-12-03T05:16:57,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c, entries=150, sequenceid=96, filesize=11.7 K 2024-12-03T05:16:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6c42aa2b936b4a2c86d1440a2f51446a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a 2024-12-03T05:16:57,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a, entries=150, sequenceid=96, filesize=11.7 K 2024-12-03T05:16:57,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 6596af7287e3b5c33fe33f24d375a15b in 635ms, sequenceid=96, compaction requested=true 2024-12-03T05:16:57,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:57,348 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:57,349 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:16:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:57,352 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:57,352 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:16:57,352 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,352 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=51.4 K 2024-12-03T05:16:57,352 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:57,352 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:16:57,353 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,353 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=46.9 K 2024-12-03T05:16:57,355 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbad58e7916c460ba66f742a15e91b19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733203013406 2024-12-03T05:16:57,355 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 329a580cdf6f460fbf383b2a7b42e513, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733203013406 2024-12-03T05:16:57,355 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5fe59da94c45f48c19efe0984adec5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733203013484 2024-12-03T05:16:57,355 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d474537d32f147c1a6967e969d41c27f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733203013484 2024-12-03T05:16:57,364 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5825f809fa0e4991a932c2fe48478465, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733203014135 2024-12-03T05:16:57,365 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b53300b5084ba983012ce0cca2b26a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733203014135 2024-12-03T05:16:57,373 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5839b0ae8ab744eda62a887ffe9dc95c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015473 2024-12-03T05:16:57,373 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a22c02e098804ee985a2a9da61a8a979, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015447 2024-12-03T05:16:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:57,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:16:57,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:57,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:57,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/224321f9682648beb007e677eea8b143 is 50, key is test_row_0/A:col10/1733203016741/Put/seqid=0 2024-12-03T05:16:57,479 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:57,480 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/e362ecaede334ff895d82fe3de30d784 is 50, key is test_row_0/B:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:57,489 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:57,489 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/cb4d234bec274165916a97ef5deb204f is 50, key is test_row_0/A:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:57,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742304_1480 (size=12001) 2024-12-03T05:16:57,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/224321f9682648beb007e677eea8b143 2024-12-03T05:16:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742305_1481 (size=12139) 2024-12-03T05:16:57,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/18dc88ef7b7e4af3b3c8bd1b42195866 is 50, key is test_row_0/B:col10/1733203016741/Put/seqid=0 2024-12-03T05:16:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742306_1482 (size=12139) 2024-12-03T05:16:57,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733203017483, max timestamp=1733203017552Num uniques 859; 2024-12-03T05:16:57,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:57,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733203017483, max timestamp=1733203017552Num uniques 859; 2024-12-03T05:16:57,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:57,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733203017483, max timestamp=1733203017552Num uniques 859; 2024-12-03T05:16:57,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:16:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T05:16:57,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:57,577 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/cb4d234bec274165916a97ef5deb204f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/cb4d234bec274165916a97ef5deb204f 2024-12-03T05:16:57,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T05:16:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:57,581 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:57,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:57,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:57,586 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into cb4d234bec274165916a97ef5deb204f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:57,586 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:57,586 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=12, startTime=1733203017348; duration=0sec 2024-12-03T05:16:57,586 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:16:57,586 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:16:57,586 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:16:57,587 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:16:57,587 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:16:57,587 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,587 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=46.9 K 2024-12-03T05:16:57,589 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c9fb49d103489d876e1737c2cc5556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733203013406 2024-12-03T05:16:57,589 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a7efc1fbd44776b513f4d0ae2cccb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733203013484 2024-12-03T05:16:57,589 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5490d01057f342d09c5599af577d5016, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733203014135 2024-12-03T05:16:57,590 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c42aa2b936b4a2c86d1440a2f51446a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015473 2024-12-03T05:16:57,601 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:16:57,602 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/662de95187534d59aa1e6a867be6edaf is 50, key is test_row_0/C:col10/1733203015473/Put/seqid=0 2024-12-03T05:16:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742307_1483 (size=12001) 2024-12-03T05:16:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742308_1484 (size=12139) 2024-12-03T05:16:57,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203077637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203077637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203077642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203077645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203077647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:57,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203077785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203077786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203077787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203077797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203077800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:57,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:57,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:57,926 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/e362ecaede334ff895d82fe3de30d784 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e362ecaede334ff895d82fe3de30d784 2024-12-03T05:16:57,931 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into e362ecaede334ff895d82fe3de30d784(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:57,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:57,931 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=12, startTime=1733203017349; duration=0sec 2024-12-03T05:16:57,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:57,931 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:16:58,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/18dc88ef7b7e4af3b3c8bd1b42195866 2024-12-03T05:16:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203078011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203078011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203078011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/7ad8a28dd3ba473eac844aa954501737 is 50, key is test_row_0/C:col10/1733203016741/Put/seqid=0 2024-12-03T05:16:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203078031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,074 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/662de95187534d59aa1e6a867be6edaf as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/662de95187534d59aa1e6a867be6edaf 2024-12-03T05:16:58,078 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into 662de95187534d59aa1e6a867be6edaf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:16:58,078 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:58,079 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=12, startTime=1733203017349; duration=0sec 2024-12-03T05:16:58,079 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:16:58,079 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:16:58,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742309_1485 (size=12001) 2024-12-03T05:16:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:58,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203078330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203078331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203078331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203078340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203078350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:58,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:58,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/7ad8a28dd3ba473eac844aa954501737 2024-12-03T05:16:58,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/224321f9682648beb007e677eea8b143 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143 2024-12-03T05:16:58,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143, entries=150, sequenceid=107, filesize=11.7 K 2024-12-03T05:16:58,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/18dc88ef7b7e4af3b3c8bd1b42195866 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866 2024-12-03T05:16:58,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866, entries=150, sequenceid=107, filesize=11.7 K 2024-12-03T05:16:58,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/7ad8a28dd3ba473eac844aa954501737 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737 2024-12-03T05:16:58,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737, entries=150, sequenceid=107, filesize=11.7 K 2024-12-03T05:16:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6596af7287e3b5c33fe33f24d375a15b in 1115ms, sequenceid=107, compaction requested=false 2024-12-03T05:16:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:16:58,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T05:16:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:58,673 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:16:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4e4fd79e30e04bf4b8404e5e4d89bf2e is 50, key is test_row_0/A:col10/1733203017633/Put/seqid=0 2024-12-03T05:16:58,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742310_1486 (size=12151) 2024-12-03T05:16:58,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4e4fd79e30e04bf4b8404e5e4d89bf2e 2024-12-03T05:16:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/335ad8839aa44747bb50a2724b02f53d is 50, key is test_row_0/B:col10/1733203017633/Put/seqid=0 2024-12-03T05:16:58,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742311_1487 (size=12151) 2024-12-03T05:16:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:16:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:16:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203078878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203078879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203078880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203078880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203078887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:58,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203078991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203079000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203079000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203079001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203079011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203079200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203079221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203079221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203079221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203079230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,282 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/335ad8839aa44747bb50a2724b02f53d 2024-12-03T05:16:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6b6973b2cd994ff99539a65c28bf862e is 50, key is test_row_0/C:col10/1733203017633/Put/seqid=0 2024-12-03T05:16:59,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742312_1488 (size=12151) 2024-12-03T05:16:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203079510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203079530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203079540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203079541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203079550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:16:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:16:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:16:59,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6b6973b2cd994ff99539a65c28bf862e 2024-12-03T05:16:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4e4fd79e30e04bf4b8404e5e4d89bf2e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e 2024-12-03T05:16:59,697 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T05:16:59,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/335ad8839aa44747bb50a2724b02f53d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d 2024-12-03T05:16:59,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T05:16:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/6b6973b2cd994ff99539a65c28bf862e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e 2024-12-03T05:16:59,704 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T05:16:59,704 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 6596af7287e3b5c33fe33f24d375a15b in 1031ms, sequenceid=144, compaction requested=true 2024-12-03T05:16:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:16:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T05:16:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T05:16:59,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T05:16:59,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1240 sec 2024-12-03T05:16:59,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1290 sec 2024-12-03T05:16:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T05:16:59,730 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:16:59,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:16:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T05:16:59,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:16:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:16:59,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:16:59,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:16:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:16:59,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:16:59,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:16:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:16:59,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/79492ddf90cc4b20ae99183a1dfb1377 is 50, key is test_row_0/A:col10/1733203018879/Put/seqid=0 2024-12-03T05:16:59,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742313_1489 (size=12151) 2024-12-03T05:16:59,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/79492ddf90cc4b20ae99183a1dfb1377 2024-12-03T05:16:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f234c691b104461a9ccb1b7d715634bd is 50, key is test_row_0/B:col10/1733203018879/Put/seqid=0 2024-12-03T05:16:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742314_1490 (size=12151) 2024-12-03T05:17:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:00,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:17:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203080075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203020069, max timestamp=1733203020086Num uniques 859; 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203020069, max timestamp=1733203020086Num uniques 859; 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203020069, max timestamp=1733203020086Num uniques 859; 2024-12-03T05:17:00,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203080113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203080114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203080119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203080120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203080200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203080230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203080230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203080230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203080230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,315 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f234c691b104461a9ccb1b7d715634bd 2024-12-03T05:17:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/fce3c7514eba4acabcd59bde0cd7755a is 50, key is test_row_0/C:col10/1733203018879/Put/seqid=0 2024-12-03T05:17:00,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742315_1491 (size=12151) 2024-12-03T05:17:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:17:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203080411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203080440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203080440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203080440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203080440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203080720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,731 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/fce3c7514eba4acabcd59bde0cd7755a 2024-12-03T05:17:00,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/79492ddf90cc4b20ae99183a1dfb1377 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377 2024-12-03T05:17:00,736 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T05:17:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f234c691b104461a9ccb1b7d715634bd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd 2024-12-03T05:17:00,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T05:17:00,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/fce3c7514eba4acabcd59bde0cd7755a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a 2024-12-03T05:17:00,742 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T05:17:00,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6596af7287e3b5c33fe33f24d375a15b in 858ms, sequenceid=155, compaction requested=true 2024-12-03T05:17:00,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:00,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:00,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T05:17:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T05:17:00,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T05:17:00,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0120 sec 2024-12-03T05:17:00,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0150 sec 2024-12-03T05:17:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:00,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:00,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:00,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/176461fbb5e446288cb04e4583d48fb5 is 50, key is test_row_0/A:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742316_1492 (size=14541) 2024-12-03T05:17:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203080784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203080782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203080785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203080786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T05:17:00,873 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:00,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T05:17:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:00,876 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:00,877 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:00,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203080912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203080912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203080913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203080913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:01,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T05:17:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:01,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203081130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203081130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203081130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203081130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/176461fbb5e446288cb04e4583d48fb5 2024-12-03T05:17:01,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/04c0c93ea4da4688948b61b5cc500764 is 50, key is test_row_0/B:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742317_1493 (size=12151) 2024-12-03T05:17:01,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/04c0c93ea4da4688948b61b5cc500764 2024-12-03T05:17:01,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T05:17:01,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:01,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/c8bdd01f678a4999bf632aef77c9cadd is 50, key is test_row_0/C:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:01,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742318_1494 (size=12151) 2024-12-03T05:17:01,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/c8bdd01f678a4999bf632aef77c9cadd 2024-12-03T05:17:01,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/176461fbb5e446288cb04e4583d48fb5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5 2024-12-03T05:17:01,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5, entries=200, sequenceid=190, filesize=14.2 K 2024-12-03T05:17:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203081231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/04c0c93ea4da4688948b61b5cc500764 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764 2024-12-03T05:17:01,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T05:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/c8bdd01f678a4999bf632aef77c9cadd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd 2024-12-03T05:17:01,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd, entries=150, sequenceid=190, filesize=11.9 K 2024-12-03T05:17:01,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 6596af7287e3b5c33fe33f24d375a15b in 482ms, sequenceid=190, compaction requested=true 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:01,256 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:01,256 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:01,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:01,259 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62983 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:01,259 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:01,259 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,259 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/cb4d234bec274165916a97ef5deb204f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=61.5 K 2024-12-03T05:17:01,259 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb4d234bec274165916a97ef5deb204f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015473 2024-12-03T05:17:01,259 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:01,259 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:01,259 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,259 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e362ecaede334ff895d82fe3de30d784, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=59.2 K 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224321f9682648beb007e677eea8b143, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733203016729 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e362ecaede334ff895d82fe3de30d784, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015473 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4fd79e30e04bf4b8404e5e4d89bf2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733203017633 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dc88ef7b7e4af3b3c8bd1b42195866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733203016729 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79492ddf90cc4b20ae99183a1dfb1377, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733203018860 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 335ad8839aa44747bb50a2724b02f53d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733203017633 2024-12-03T05:17:01,260 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f234c691b104461a9ccb1b7d715634bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733203018860 2024-12-03T05:17:01,261 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176461fbb5e446288cb04e4583d48fb5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020096 2024-12-03T05:17:01,261 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c0c93ea4da4688948b61b5cc500764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020112 2024-12-03T05:17:01,283 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:01,283 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:01,284 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/e6022eb946bb4d0181eda9cf1f2cbbb7 is 50, key is test_row_0/B:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:01,284 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/37fe445d817a4b06b2f28de7a2b3a803 is 50, key is test_row_0/A:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:01,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742320_1496 (size=12459) 2024-12-03T05:17:01,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742319_1495 (size=12459) 2024-12-03T05:17:01,337 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/e6022eb946bb4d0181eda9cf1f2cbbb7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e6022eb946bb4d0181eda9cf1f2cbbb7 2024-12-03T05:17:01,345 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into e6022eb946bb4d0181eda9cf1f2cbbb7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:01,345 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:01,345 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=11, startTime=1733203021256; duration=0sec 2024-12-03T05:17:01,345 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:01,345 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:01,345 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:01,346 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:01,346 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:01,346 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,347 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/662de95187534d59aa1e6a867be6edaf, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=59.2 K 2024-12-03T05:17:01,347 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 662de95187534d59aa1e6a867be6edaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733203015473 2024-12-03T05:17:01,347 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad8a28dd3ba473eac844aa954501737, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733203016729 2024-12-03T05:17:01,347 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6973b2cd994ff99539a65c28bf862e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733203017633 2024-12-03T05:17:01,348 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fce3c7514eba4acabcd59bde0cd7755a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733203018860 2024-12-03T05:17:01,348 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c8bdd01f678a4999bf632aef77c9cadd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020112 2024-12-03T05:17:01,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:01,359 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:01,369 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#413 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:01,370 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/bbf1d47e11ba4801944463e13dee482e is 50, key is test_row_0/C:col10/1733203020772/Put/seqid=0 2024-12-03T05:17:01,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2b925ebbc82047d29fb7da0556ddccd0 is 50, key is test_row_1/A:col10/1733203020783/Put/seqid=0 2024-12-03T05:17:01,377 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/37fe445d817a4b06b2f28de7a2b3a803 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/37fe445d817a4b06b2f28de7a2b3a803 2024-12-03T05:17:01,384 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 37fe445d817a4b06b2f28de7a2b3a803(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:01,384 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:01,384 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=11, startTime=1733203021256; duration=0sec 2024-12-03T05:17:01,384 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:01,384 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:01,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742321_1497 (size=12459) 2024-12-03T05:17:01,415 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/bbf1d47e11ba4801944463e13dee482e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/bbf1d47e11ba4801944463e13dee482e 2024-12-03T05:17:01,420 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into bbf1d47e11ba4801944463e13dee482e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:01,420 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:01,420 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=11, startTime=1733203021256; duration=0sec 2024-12-03T05:17:01,420 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:01,420 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:01,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742322_1498 (size=9757) 2024-12-03T05:17:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203021497, max timestamp=1733203021512Num uniques 859; 2024-12-03T05:17:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203021497, max timestamp=1733203021512Num uniques 859; 2024-12-03T05:17:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:01,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203021497, max timestamp=1733203021512Num uniques 859; 2024-12-03T05:17:01,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203081562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203081573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203081578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203081578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203081691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203081701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203081710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203081710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2b925ebbc82047d29fb7da0556ddccd0 2024-12-03T05:17:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3a9cc336134e4198a716da8824a63c7c is 50, key is test_row_1/B:col10/1733203020783/Put/seqid=0 2024-12-03T05:17:01,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742323_1499 (size=9757) 2024-12-03T05:17:01,864 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3a9cc336134e4198a716da8824a63c7c 2024-12-03T05:17:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/05b0f19b9b9744e5a2b93fdd439b3ed5 is 50, key is test_row_1/C:col10/1733203020783/Put/seqid=0 2024-12-03T05:17:01,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203081904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203081921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203081931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203081931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:01,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742324_1500 (size=9757) 2024-12-03T05:17:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:02,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203082220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203082232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203082250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203082252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203082258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,284 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,351 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/05b0f19b9b9744e5a2b93fdd439b3ed5 2024-12-03T05:17:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2b925ebbc82047d29fb7da0556ddccd0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0 2024-12-03T05:17:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:17:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3a9cc336134e4198a716da8824a63c7c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c 2024-12-03T05:17:02,380 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:17:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/05b0f19b9b9744e5a2b93fdd439b3ed5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5 2024-12-03T05:17:02,399 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5, entries=100, sequenceid=201, filesize=9.5 K 2024-12-03T05:17:02,401 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 6596af7287e3b5c33fe33f24d375a15b in 1042ms, sequenceid=201, compaction requested=false 2024-12-03T05:17:02,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:02,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:02,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T05:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T05:17:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T05:17:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5370 sec 2024-12-03T05:17:02,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5550 sec 2024-12-03T05:17:02,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203021558, max timestamp=1733203022756Num uniques 1068; 2024-12-03T05:17:02,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T05:17:02,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203021558, max timestamp=1733203022756Num uniques 1068; 2024-12-03T05:17:02,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T05:17:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:17:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:02,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/ed2b65c587f240ae8fa4251c15eb7cd9 is 50, key is test_row_0/A:col10/1733203021558/Put/seqid=0 2024-12-03T05:17:02,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742325_1501 (size=14541) 2024-12-03T05:17:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203082838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203082840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203082844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203082846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203082962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203082971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203082971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203082971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T05:17:03,021 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:03,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T05:17:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:03,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:03,033 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:03,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/ed2b65c587f240ae8fa4251c15eb7cd9 2024-12-03T05:17:03,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203083190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203083192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203083193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/76811f12e960494790425e36d8f1de7d is 50, key is test_row_0/B:col10/1733203021558/Put/seqid=0 2024-12-03T05:17:03,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742326_1502 (size=12151) 2024-12-03T05:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:03,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:03,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203083492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203083511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203083511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203083511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:03,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:03,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/76811f12e960494790425e36d8f1de7d 2024-12-03T05:17:03,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/30426846a90b47deb69f7fe82648e695 is 50, key is test_row_0/C:col10/1733203021558/Put/seqid=0 2024-12-03T05:17:03,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742327_1503 (size=12151) 2024-12-03T05:17:03,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:03,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:03,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:03,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:04,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:04,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:04,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203084011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203084031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203084031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203084031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/30426846a90b47deb69f7fe82648e695 2024-12-03T05:17:04,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/ed2b65c587f240ae8fa4251c15eb7cd9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9 2024-12-03T05:17:04,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9, entries=200, sequenceid=241, filesize=14.2 K 2024-12-03T05:17:04,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/76811f12e960494790425e36d8f1de7d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d 2024-12-03T05:17:04,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:17:04,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/30426846a90b47deb69f7fe82648e695 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695 2024-12-03T05:17:04,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T05:17:04,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for 6596af7287e3b5c33fe33f24d375a15b in 1373ms, sequenceid=241, compaction requested=true 2024-12-03T05:17:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:04,142 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:04,142 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:04,143 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:04,143 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:04,143 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,143 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/37fe445d817a4b06b2f28de7a2b3a803, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=35.9 K 2024-12-03T05:17:04,143 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:04,143 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:04,143 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,144 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e6022eb946bb4d0181eda9cf1f2cbbb7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=33.6 K 2024-12-03T05:17:04,144 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fe445d817a4b06b2f28de7a2b3a803, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020112 2024-12-03T05:17:04,144 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting e6022eb946bb4d0181eda9cf1f2cbbb7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020112 2024-12-03T05:17:04,144 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b925ebbc82047d29fb7da0556ddccd0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733203020783 2024-12-03T05:17:04,144 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2b65c587f240ae8fa4251c15eb7cd9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733203021558 2024-12-03T05:17:04,144 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9cc336134e4198a716da8824a63c7c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733203020783 2024-12-03T05:17:04,145 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 76811f12e960494790425e36d8f1de7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733203021558 2024-12-03T05:17:04,149 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:04,150 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/63f975f349284dabbd0d20f6ca18cab5 is 50, key is test_row_0/A:col10/1733203021558/Put/seqid=0 2024-12-03T05:17:04,151 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:04,152 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a9e1115a4eb9454281b95a4dab695b2f is 50, key is test_row_0/B:col10/1733203021558/Put/seqid=0 2024-12-03T05:17:04,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T05:17:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:04,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/032bf0627f554ae7b1fa8516031a7c1a is 50, key is test_row_0/A:col10/1733203022842/Put/seqid=0 2024-12-03T05:17:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742329_1505 (size=12561) 2024-12-03T05:17:04,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742328_1504 (size=12561) 2024-12-03T05:17:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742330_1506 (size=12251) 2024-12-03T05:17:04,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/032bf0627f554ae7b1fa8516031a7c1a 2024-12-03T05:17:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/14688510733e466e9e6953a7af061030 is 50, key is test_row_0/B:col10/1733203022842/Put/seqid=0 2024-12-03T05:17:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742331_1507 (size=12251) 2024-12-03T05:17:04,257 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/14688510733e466e9e6953a7af061030 2024-12-03T05:17:04,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/deab131d0a5c4f09aa1bcbd73fb3ab14 is 50, key is test_row_0/C:col10/1733203022842/Put/seqid=0 2024-12-03T05:17:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:04,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:04,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742332_1508 (size=12251) 2024-12-03T05:17:04,344 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/deab131d0a5c4f09aa1bcbd73fb3ab14 2024-12-03T05:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/032bf0627f554ae7b1fa8516031a7c1a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a 2024-12-03T05:17:04,371 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a, entries=150, sequenceid=261, filesize=12.0 K 2024-12-03T05:17:04,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/14688510733e466e9e6953a7af061030 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030 2024-12-03T05:17:04,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030, entries=150, sequenceid=261, filesize=12.0 K 2024-12-03T05:17:04,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/deab131d0a5c4f09aa1bcbd73fb3ab14 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14 2024-12-03T05:17:04,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203084381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,395 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14, entries=150, sequenceid=261, filesize=12.0 K 2024-12-03T05:17:04,396 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6596af7287e3b5c33fe33f24d375a15b in 240ms, sequenceid=261, compaction requested=true 2024-12-03T05:17:04,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T05:17:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T05:17:04,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T05:17:04,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3640 sec 2024-12-03T05:17:04,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3780 sec 2024-12-03T05:17:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:04,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:04,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:04,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2a43c399b0984965afdf2b40eaba246e is 50, key is test_row_0/A:col10/1733203024379/Put/seqid=0 2024-12-03T05:17:04,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742333_1509 (size=12301) 2024-12-03T05:17:04,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2a43c399b0984965afdf2b40eaba246e 2024-12-03T05:17:04,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/fa7d6150e092459cb387b59d58b0e91e is 50, key is test_row_0/B:col10/1733203024379/Put/seqid=0 2024-12-03T05:17:04,602 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/63f975f349284dabbd0d20f6ca18cab5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/63f975f349284dabbd0d20f6ca18cab5 2024-12-03T05:17:04,612 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 63f975f349284dabbd0d20f6ca18cab5(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:04,612 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:04,612 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203024142; duration=0sec 2024-12-03T05:17:04,612 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:04,612 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:04,613 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:17:04,614 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a9e1115a4eb9454281b95a4dab695b2f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a9e1115a4eb9454281b95a4dab695b2f 2024-12-03T05:17:04,616 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:17:04,616 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:04,616 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:04,617 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/bbf1d47e11ba4801944463e13dee482e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=45.5 K 2024-12-03T05:17:04,618 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf1d47e11ba4801944463e13dee482e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733203020112 2024-12-03T05:17:04,619 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b0f19b9b9744e5a2b93fdd439b3ed5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733203020783 2024-12-03T05:17:04,620 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30426846a90b47deb69f7fe82648e695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733203021558 2024-12-03T05:17:04,621 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting deab131d0a5c4f09aa1bcbd73fb3ab14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733203022816 2024-12-03T05:17:04,624 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into a9e1115a4eb9454281b95a4dab695b2f(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:04,624 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:04,624 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203024142; duration=0sec 2024-12-03T05:17:04,625 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:04,625 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:04,651 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:04,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742334_1510 (size=12301) 2024-12-03T05:17:04,651 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/67a8bc975a51449e987b2d6e1ba99966 is 50, key is test_row_0/C:col10/1733203022842/Put/seqid=0 2024-12-03T05:17:04,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742335_1511 (size=12695) 2024-12-03T05:17:04,684 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/67a8bc975a51449e987b2d6e1ba99966 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/67a8bc975a51449e987b2d6e1ba99966 2024-12-03T05:17:04,689 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into 67a8bc975a51449e987b2d6e1ba99966(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:04,689 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:04,689 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=12, startTime=1733203024143; duration=0sec 2024-12-03T05:17:04,689 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:04,690 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203084777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:04,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203084901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203085036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,044 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/fa7d6150e092459cb387b59d58b0e91e 2024-12-03T05:17:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203085048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203085051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203085051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,057 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,058 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,058 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/556fc563db704f0eb8bf11823ce4324a is 50, key is test_row_0/C:col10/1733203024379/Put/seqid=0 2024-12-03T05:17:05,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742336_1512 (size=12301) 2024-12-03T05:17:05,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/556fc563db704f0eb8bf11823ce4324a 2024-12-03T05:17:05,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/2a43c399b0984965afdf2b40eaba246e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e 2024-12-03T05:17:05,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203085111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e, entries=150, sequenceid=274, filesize=12.0 K 2024-12-03T05:17:05,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/fa7d6150e092459cb387b59d58b0e91e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e 2024-12-03T05:17:05,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e, entries=150, sequenceid=274, filesize=12.0 K 2024-12-03T05:17:05,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/556fc563db704f0eb8bf11823ce4324a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a 2024-12-03T05:17:05,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a, entries=150, sequenceid=274, filesize=12.0 K 2024-12-03T05:17:05,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6596af7287e3b5c33fe33f24d375a15b in 609ms, sequenceid=274, compaction requested=true 2024-12-03T05:17:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:05,131 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:05,131 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:05,132 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37113 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:05,132 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:05,132 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,132 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/63f975f349284dabbd0d20f6ca18cab5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.2 K 2024-12-03T05:17:05,133 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f975f349284dabbd0d20f6ca18cab5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733203021558 2024-12-03T05:17:05,133 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37113 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:05,133 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:05,133 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,133 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a9e1115a4eb9454281b95a4dab695b2f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.2 K 2024-12-03T05:17:05,133 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032bf0627f554ae7b1fa8516031a7c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733203022816 2024-12-03T05:17:05,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e1115a4eb9454281b95a4dab695b2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733203021558 2024-12-03T05:17:05,134 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a43c399b0984965afdf2b40eaba246e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733203024349 2024-12-03T05:17:05,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 14688510733e466e9e6953a7af061030, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733203022816 2024-12-03T05:17:05,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7d6150e092459cb387b59d58b0e91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733203024349 2024-12-03T05:17:05,146 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:05,147 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/2b0fbbbf4db24256bc4bf797d8f02eac is 50, key is test_row_0/B:col10/1733203024379/Put/seqid=0 2024-12-03T05:17:05,170 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:05,171 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/1ae88f177e5a4dafa4c126476a4e6a55 is 50, key is test_row_0/A:col10/1733203024379/Put/seqid=0 2024-12-03T05:17:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T05:17:05,184 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:05,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T05:17:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:17:05,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:05,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:05,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:05,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742337_1513 (size=12813) 2024-12-03T05:17:05,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742338_1514 (size=12813) 2024-12-03T05:17:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:17:05,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T05:17:05,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,360 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/82144988588b40fdb704e02b8d378e72 is 50, key is test_row_0/A:col10/1733203024696/Put/seqid=0 2024-12-03T05:17:05,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742339_1515 (size=12301) 2024-12-03T05:17:05,399 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/82144988588b40fdb704e02b8d378e72 2024-12-03T05:17:05,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3f57abbe7c9b48f7b7265194beda1c9c is 50, key is test_row_0/B:col10/1733203024696/Put/seqid=0 2024-12-03T05:17:05,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742340_1516 (size=12301) 2024-12-03T05:17:05,423 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3f57abbe7c9b48f7b7265194beda1c9c 2024-12-03T05:17:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/23b8e11ed87644a4a5fd791f8557c574 is 50, key is test_row_0/C:col10/1733203024696/Put/seqid=0 2024-12-03T05:17:05,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742341_1517 (size=12301) 2024-12-03T05:17:05,435 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/23b8e11ed87644a4a5fd791f8557c574 2024-12-03T05:17:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/82144988588b40fdb704e02b8d378e72 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72 2024-12-03T05:17:05,445 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T05:17:05,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3f57abbe7c9b48f7b7265194beda1c9c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c 2024-12-03T05:17:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:05,453 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T05:17:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/23b8e11ed87644a4a5fd791f8557c574 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574 2024-12-03T05:17:05,458 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T05:17:05,459 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 6596af7287e3b5c33fe33f24d375a15b in 98ms, sequenceid=296, compaction requested=true 2024-12-03T05:17:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T05:17:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T05:17:05,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T05:17:05,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 274 msec 2024-12-03T05:17:05,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 280 msec 2024-12-03T05:17:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:05,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/8a9642680b7144efb4683effa80c3a7d is 50, key is test_row_0/A:col10/1733203025465/Put/seqid=0 2024-12-03T05:17:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T05:17:05,509 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:05,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T05:17:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:05,514 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:05,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:05,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:05,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742342_1518 (size=14741) 2024-12-03T05:17:05,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/8a9642680b7144efb4683effa80c3a7d 2024-12-03T05:17:05,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/35796910cec54f2280a15463e2cd8ca1 is 50, key is test_row_0/B:col10/1733203025465/Put/seqid=0 2024-12-03T05:17:05,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742343_1519 (size=12301) 2024-12-03T05:17:05,615 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/2b0fbbbf4db24256bc4bf797d8f02eac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/2b0fbbbf4db24256bc4bf797d8f02eac 2024-12-03T05:17:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:05,627 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into 2b0fbbbf4db24256bc4bf797d8f02eac(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:05,627 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:05,627 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203025131; duration=0sec 2024-12-03T05:17:05,627 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:05,627 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:05,627 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:05,630 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:05,631 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:05,631 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,631 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/67a8bc975a51449e987b2d6e1ba99966, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.4 K 2024-12-03T05:17:05,632 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/1ae88f177e5a4dafa4c126476a4e6a55 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/1ae88f177e5a4dafa4c126476a4e6a55 2024-12-03T05:17:05,632 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a8bc975a51449e987b2d6e1ba99966, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733203022816 2024-12-03T05:17:05,633 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 556fc563db704f0eb8bf11823ce4324a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733203024349 2024-12-03T05:17:05,634 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b8e11ed87644a4a5fd791f8557c574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733203024696 2024-12-03T05:17:05,645 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 1ae88f177e5a4dafa4c126476a4e6a55(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:05,645 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:05,645 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203025131; duration=0sec 2024-12-03T05:17:05,645 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:05,645 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:05,660 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:05,661 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/35594ea7b40641a28319ee529414c250 is 50, key is test_row_0/C:col10/1733203024696/Put/seqid=0 2024-12-03T05:17:05,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742344_1520 (size=12847) 2024-12-03T05:17:05,742 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/35594ea7b40641a28319ee529414c250 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/35594ea7b40641a28319ee529414c250 2024-12-03T05:17:05,747 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into 35594ea7b40641a28319ee529414c250(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:05,747 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:05,747 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=13, startTime=1733203025132; duration=0sec 2024-12-03T05:17:05,747 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:05,747 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:05,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203025721, max timestamp=1733203025772Num uniques 859; 2024-12-03T05:17:05,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:05,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203025721, max timestamp=1733203025772Num uniques 859; 2024-12-03T05:17:05,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203025721, max timestamp=1733203025772Num uniques 859; 2024-12-03T05:17:05,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:05,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:05,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203085979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:05,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/35796910cec54f2280a15463e2cd8ca1 2024-12-03T05:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:05,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3a3e45c0713c4c45bda9d17257f48735 is 50, key is test_row_0/C:col10/1733203025465/Put/seqid=0 2024-12-03T05:17:06,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742345_1521 (size=12301) 2024-12-03T05:17:06,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3a3e45c0713c4c45bda9d17257f48735 2024-12-03T05:17:06,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/8a9642680b7144efb4683effa80c3a7d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d 2024-12-03T05:17:06,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d, entries=200, sequenceid=306, filesize=14.4 K 2024-12-03T05:17:06,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/35796910cec54f2280a15463e2cd8ca1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1 2024-12-03T05:17:06,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T05:17:06,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3a3e45c0713c4c45bda9d17257f48735 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735 2024-12-03T05:17:06,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T05:17:06,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6596af7287e3b5c33fe33f24d375a15b in 604ms, sequenceid=306, compaction requested=true 2024-12-03T05:17:06,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:06,090 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:06,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:06,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:06,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:06,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:06,091 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:06,091 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:06,091 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:06,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:06,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:06,091 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/1ae88f177e5a4dafa4c126476a4e6a55, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=38.9 K 2024-12-03T05:17:06,095 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae88f177e5a4dafa4c126476a4e6a55, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733203024349 2024-12-03T05:17:06,095 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82144988588b40fdb704e02b8d378e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733203024696 2024-12-03T05:17:06,096 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:06,096 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a9642680b7144efb4683effa80c3a7d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733203025465 2024-12-03T05:17:06,096 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:06,096 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,096 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/2b0fbbbf4db24256bc4bf797d8f02eac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.5 K 2024-12-03T05:17:06,097 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0fbbbf4db24256bc4bf797d8f02eac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733203024349 2024-12-03T05:17:06,097 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f57abbe7c9b48f7b7265194beda1c9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733203024696 2024-12-03T05:17:06,099 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 35796910cec54f2280a15463e2cd8ca1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733203025465 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203025915, max timestamp=1733203026098Num uniques 1018; 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203025915, max timestamp=1733203026098Num uniques 1018; 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203025915, max timestamp=1733203026098Num uniques 1018; 2024-12-03T05:17:06,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:06,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:06,111 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:06,111 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/7eaa4d8f747e4c20a87a46d80fd62027 is 50, key is test_row_0/A:col10/1733203025465/Put/seqid=0 2024-12-03T05:17:06,113 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:06,114 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/d82d44184e284460b1d76ba52f7ce5eb is 50, key is test_row_0/B:col10/1733203025465/Put/seqid=0 2024-12-03T05:17:06,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f9112dee62d942d69bfbdb0cef23026f is 50, key is test_row_0/A:col10/1733203025960/Put/seqid=0 2024-12-03T05:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:06,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742347_1523 (size=12915) 2024-12-03T05:17:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,157 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/d82d44184e284460b1d76ba52f7ce5eb as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d82d44184e284460b1d76ba52f7ce5eb 2024-12-03T05:17:06,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742346_1522 (size=12915) 2024-12-03T05:17:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742348_1524 (size=14741) 2024-12-03T05:17:06,169 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into d82d44184e284460b1d76ba52f7ce5eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:06,169 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:06,169 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203026090; duration=0sec 2024-12-03T05:17:06,169 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:06,169 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:06,169 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:17:06,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:17:06,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:17:06,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. because compaction request was cancelled 2024-12-03T05:17:06,171 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203026214, max timestamp=1733203026262Num uniques 859; 2024-12-03T05:17:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203026214, max timestamp=1733203026262Num uniques 859; 2024-12-03T05:17:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:06,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203026214, max timestamp=1733203026262Num uniques 859; 2024-12-03T05:17:06,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:06,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203026352, max timestamp=1733203026421Num uniques 1018; 2024-12-03T05:17:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203026352, max timestamp=1733203026421Num uniques 1018; 2024-12-03T05:17:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203026352, max timestamp=1733203026421Num uniques 1018; 2024-12-03T05:17:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:06,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733203026515, max timestamp=1733203026561Num uniques 1027; 2024-12-03T05:17:06,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T05:17:06,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733203026515, max timestamp=1733203026561Num uniques 1027; 2024-12-03T05:17:06,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T05:17:06,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733203026515, max timestamp=1733203026561Num uniques 1027; 2024-12-03T05:17:06,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T05:17:06,569 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/7eaa4d8f747e4c20a87a46d80fd62027 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7eaa4d8f747e4c20a87a46d80fd62027 2024-12-03T05:17:06,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f9112dee62d942d69bfbdb0cef23026f 2024-12-03T05:17:06,573 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 7eaa4d8f747e4c20a87a46d80fd62027(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:06,573 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:06,573 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203026090; duration=0sec 2024-12-03T05:17:06,573 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:06,573 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:06,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/7dcad4ee0d7649afbce28ec35957fba2 is 50, key is test_row_0/B:col10/1733203025960/Put/seqid=0 2024-12-03T05:17:06,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742349_1525 (size=12301) 2024-12-03T05:17:06,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203026599, max timestamp=1733203026715Num uniques 1036; 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203026599, max timestamp=1733203026715Num uniques 1036; 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203026599, max timestamp=1733203026715Num uniques 1036; 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:17:06,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T05:17:06,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:06,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:06,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733203026876, max timestamp=1733203026920Num uniques 1045; 2024-12-03T05:17:06,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:17:06,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733203026876, max timestamp=1733203026920Num uniques 1045; 2024-12-03T05:17:06,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733203026876, max timestamp=1733203026920Num uniques 1045; 2024-12-03T05:17:06,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:17:06,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-03T05:17:06,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/7dcad4ee0d7649afbce28ec35957fba2 2024-12-03T05:17:06,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 is 50, key is test_row_0/C:col10/1733203025960/Put/seqid=0 2024-12-03T05:17:07,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742350_1526 (size=12301) 2024-12-03T05:17:07,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 2024-12-03T05:17:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f9112dee62d942d69bfbdb0cef23026f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f 2024-12-03T05:17:07,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f, entries=200, sequenceid=347, filesize=14.4 K 2024-12-03T05:17:07,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/7dcad4ee0d7649afbce28ec35957fba2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2 2024-12-03T05:17:07,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2, entries=150, sequenceid=347, filesize=12.0 K 2024-12-03T05:17:07,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 2024-12-03T05:17:07,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7, entries=150, sequenceid=347, filesize=12.0 K 2024-12-03T05:17:07,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=134.18 KB/137400 for 6596af7287e3b5c33fe33f24d375a15b in 1030ms, sequenceid=347, compaction requested=true 2024-12-03T05:17:07,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:07,133 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T05:17:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=140.89 KB heapSize=401.53 KB 2024-12-03T05:17:07,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:17:07,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:17:07,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. because compaction request was cancelled 2024-12-03T05:17:07,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:07,134 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:07,136 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:17:07,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733203027066, max timestamp=1733203027133Num uniques 1054; 2024-12-03T05:17:07,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-12-03T05:17:07,142 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:07,142 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:07,142 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,142 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/35594ea7b40641a28319ee529414c250, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.6 K 2024-12-03T05:17:07,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733203027066, max timestamp=1733203027133Num uniques 1054; 2024-12-03T05:17:07,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-12-03T05:17:07,148 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:17:07,148 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:17:07,148 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. because compaction request was cancelled 2024-12-03T05:17:07,149 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:07,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733203027099, max timestamp=1733203027133Num uniques 1054; 2024-12-03T05:17:07,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-03T05:17:07,152 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 35594ea7b40641a28319ee529414c250, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733203024696 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:07,153 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3e45c0713c4c45bda9d17257f48735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733203025465 2024-12-03T05:17:07,157 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef1a531fd3c4c24afcf19ebb5f3e0c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733203025915 2024-12-03T05:17:07,189 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:07,190 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d654566568fd47d391ad958d359fd32c is 50, key is test_row_0/C:col10/1733203025960/Put/seqid=0 2024-12-03T05:17:07,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/e9da728427e34c22a3ea6f1a90028cc6 is 50, key is test_row_0/A:col10/1733203027133/Put/seqid=0 2024-12-03T05:17:07,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742351_1527 (size=12949) 2024-12-03T05:17:07,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,241 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d654566568fd47d391ad958d359fd32c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d654566568fd47d391ad958d359fd32c 2024-12-03T05:17:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742352_1528 (size=12301) 2024-12-03T05:17:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/e9da728427e34c22a3ea6f1a90028cc6 2024-12-03T05:17:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,247 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into d654566568fd47d391ad958d359fd32c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:07,247 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:07,247 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=13, startTime=1733203027133; duration=0sec 2024-12-03T05:17:07,247 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:07,247 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:07,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733203027206, max timestamp=1733203027257Num uniques 859; 2024-12-03T05:17:07,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203027229, max timestamp=1733203027257Num uniques 859; 2024-12-03T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203027229, max timestamp=1733203027257Num uniques 859; 2024-12-03T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:07,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:07,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/af010a405d514cb2a5ddfca7fb101a12 is 50, key is test_row_0/B:col10/1733203027133/Put/seqid=0 2024-12-03T05:17:07,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742353_1529 (size=12301) 2024-12-03T05:17:07,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203087333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203087338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203027330, max timestamp=1733203027342Num uniques 1018; 2024-12-03T05:17:07,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203027330, max timestamp=1733203027342Num uniques 1168; 2024-12-03T05:17:07,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-03T05:17:07,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:07,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7682, totalHeapSize=17712, min timestamp=1733203027330, max timestamp=1733203027348Num uniques 1018; 2024-12-03T05:17:07,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46612, new segment data size=7682, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:07,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733203027428, max timestamp=1733203027459Num uniques 1027; 2024-12-03T05:17:07,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T05:17:07,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733203027428, max timestamp=1733203027459Num uniques 1027; 2024-12-03T05:17:07,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T05:17:07,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733203027428, max timestamp=1733203027459Num uniques 1027; 2024-12-03T05:17:07,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T05:17:07,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203027545, max timestamp=1733203027587Num uniques 1136; 2024-12-03T05:17:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-03T05:17:07,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203027545, max timestamp=1733203027587Num uniques 1136; 2024-12-03T05:17:07,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-03T05:17:07,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733203027545, max timestamp=1733203027587Num uniques 1136; 2024-12-03T05:17:07,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-03T05:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:07,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733203027614, max timestamp=1733203027666Num uniques 1045; 2024-12-03T05:17:07,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T05:17:07,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733203027614, max timestamp=1733203027666Num uniques 1045; 2024-12-03T05:17:07,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T05:17:07,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733203027614, max timestamp=1733203027666Num uniques 1045; 2024-12-03T05:17:07,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-12-03T05:17:07,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/af010a405d514cb2a5ddfca7fb101a12 2024-12-03T05:17:07,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733203027687, max timestamp=1733203027731Num uniques 1104; 2024-12-03T05:17:07,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-12-03T05:17:07,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733203027687, max timestamp=1733203027731Num uniques 1104; 2024-12-03T05:17:07,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-12-03T05:17:07,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1733203027687, max timestamp=1733203027731Num uniques 1009; 2024-12-03T05:17:07,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=16210, suffix heap size=132904,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=354 2024-12-03T05:17:07,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/40ea54a66c6a41d0ab3a03b992eb6aac is 50, key is test_row_0/C:col10/1733203027133/Put/seqid=0 2024-12-03T05:17:07,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742354_1530 (size=12301) 2024-12-03T05:17:07,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203087824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203087826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203087837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203087838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203087839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:07,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:07,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203087950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203087950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203087961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203087961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203087961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:08,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:08,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/40ea54a66c6a41d0ab3a03b992eb6aac 2024-12-03T05:17:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203088161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203088161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/e9da728427e34c22a3ea6f1a90028cc6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6 2024-12-03T05:17:08,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6, entries=150, sequenceid=456, filesize=12.0 K 2024-12-03T05:17:08,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/af010a405d514cb2a5ddfca7fb101a12 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12 2024-12-03T05:17:08,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12, entries=150, sequenceid=456, filesize=12.0 K 2024-12-03T05:17:08,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/40ea54a66c6a41d0ab3a03b992eb6aac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac 2024-12-03T05:17:08,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac, entries=150, sequenceid=456, filesize=12.0 K 2024-12-03T05:17:08,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~97.13 KB/99456, currentSize=161.02 KB/164880 for 6596af7287e3b5c33fe33f24d375a15b in 1064ms, sequenceid=456, compaction requested=true 2024-12-03T05:17:08,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:08,198 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:08,198 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:08,199 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:08,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:08,199 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:08,199 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:08,199 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,199 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,199 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7eaa4d8f747e4c20a87a46d80fd62027, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=39.0 K 2024-12-03T05:17:08,199 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d82d44184e284460b1d76ba52f7ce5eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.6 K 2024-12-03T05:17:08,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eaa4d8f747e4c20a87a46d80fd62027, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733203025465 2024-12-03T05:17:08,200 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d82d44184e284460b1d76ba52f7ce5eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733203025465 2024-12-03T05:17:08,200 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9112dee62d942d69bfbdb0cef23026f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733203025915 2024-12-03T05:17:08,200 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dcad4ee0d7649afbce28ec35957fba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733203025915 2024-12-03T05:17:08,204 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9da728427e34c22a3ea6f1a90028cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733203027099 2024-12-03T05:17:08,208 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting af010a405d514cb2a5ddfca7fb101a12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733203027099 2024-12-03T05:17:08,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T05:17:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=161.02 KB heapSize=446.39 KB 2024-12-03T05:17:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:08,214 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:08,215 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/6769045f381b4c8d861772124f5a1e2d is 50, key is test_row_0/A:col10/1733203027133/Put/seqid=0 2024-12-03T05:17:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:08,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:08,225 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:08,225 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a105e13f6ebc4966b4523d7607b71c0d is 50, key is test_row_0/B:col10/1733203027133/Put/seqid=0 2024-12-03T05:17:08,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/bc75ca57c395485dbe54a08c19492e76 is 50, key is test_row_0/A:col10/1733203027837/Put/seqid=0 2024-12-03T05:17:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203088225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203088229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742356_1532 (size=13017) 2024-12-03T05:17:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742357_1533 (size=12301) 2024-12-03T05:17:08,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742355_1531 (size=13017) 2024-12-03T05:17:08,253 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a105e13f6ebc4966b4523d7607b71c0d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a105e13f6ebc4966b4523d7607b71c0d 2024-12-03T05:17:08,258 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into a105e13f6ebc4966b4523d7607b71c0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:08,258 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:08,258 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203028198; duration=0sec 2024-12-03T05:17:08,258 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:08,258 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:08,258 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:17:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203088241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,261 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T05:17:08,261 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T05:17:08,261 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. because compaction request was cancelled 2024-12-03T05:17:08,261 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203088352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203088352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203088371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203088481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203088481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203088570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203088570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203088581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:08,648 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/bc75ca57c395485dbe54a08c19492e76 2024-12-03T05:17:08,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/480dad16a83b4ced932f057d11e6805e is 50, key is test_row_0/B:col10/1733203027837/Put/seqid=0 2024-12-03T05:17:08,657 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/6769045f381b4c8d861772124f5a1e2d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6769045f381b4c8d861772124f5a1e2d 2024-12-03T05:17:08,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742358_1534 (size=12301) 2024-12-03T05:17:08,658 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/480dad16a83b4ced932f057d11e6805e 2024-12-03T05:17:08,662 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 6769045f381b4c8d861772124f5a1e2d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:08,662 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:08,662 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203028198; duration=0sec 2024-12-03T05:17:08,662 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:08,662 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3d1275ea33ce42bebf9db12d2febcdf5 is 50, key is test_row_0/C:col10/1733203027837/Put/seqid=0 2024-12-03T05:17:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742359_1535 (size=12301) 2024-12-03T05:17:08,699 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3d1275ea33ce42bebf9db12d2febcdf5 2024-12-03T05:17:08,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/bc75ca57c395485dbe54a08c19492e76 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76 2024-12-03T05:17:08,709 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76, entries=150, sequenceid=580, filesize=12.0 K 2024-12-03T05:17:08,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/480dad16a83b4ced932f057d11e6805e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e 2024-12-03T05:17:08,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e, entries=150, sequenceid=580, filesize=12.0 K 2024-12-03T05:17:08,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/3d1275ea33ce42bebf9db12d2febcdf5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5 2024-12-03T05:17:08,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5, entries=150, sequenceid=580, filesize=12.0 K 2024-12-03T05:17:08,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~446.34 KB/457056, currentSize=26.84 KB/27480 for 6596af7287e3b5c33fe33f24d375a15b in 517ms, sequenceid=580, compaction requested=true 2024-12-03T05:17:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T05:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T05:17:08,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T05:17:08,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2240 sec 2024-12-03T05:17:08,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.2330 sec 2024-12-03T05:17:08,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:17:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:08,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a3892cbb21d246809427b77af145e523 is 50, key is test_row_0/A:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:08,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742360_1536 (size=14741) 2024-12-03T05:17:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203089005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203028975, max timestamp=1733203029017Num uniques 859; 2024-12-03T05:17:09,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203028975, max timestamp=1733203029017Num uniques 859; 2024-12-03T05:17:09,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:09,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:09,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203028975, max timestamp=1733203029017Num uniques 859; 2024-12-03T05:17:09,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203089065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203089074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203089084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203089085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203089190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203089191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203089210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203089210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a3892cbb21d246809427b77af145e523 2024-12-03T05:17:09,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/159872d0fd6e4176825bf2a60148f896 is 50, key is test_row_0/B:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742361_1537 (size=12301) 2024-12-03T05:17:09,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203089401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203089401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203089430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203089430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T05:17:09,681 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:09,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T05:17:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:17:09,697 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:09,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:09,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203089720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203089720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203089740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203089750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:17:09,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/159872d0fd6e4176825bf2a60148f896 2024-12-03T05:17:09,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/e401f5402bdb4e8e8d9d9af4a1c008e5 is 50, key is test_row_0/C:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:09,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:17:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:09,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:09,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742362_1538 (size=12301) 2024-12-03T05:17:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:09,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/e401f5402bdb4e8e8d9d9af4a1c008e5 2024-12-03T05:17:09,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/a3892cbb21d246809427b77af145e523 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523 2024-12-03T05:17:09,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523, entries=200, sequenceid=593, filesize=14.4 K 2024-12-03T05:17:09,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/159872d0fd6e4176825bf2a60148f896 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896 2024-12-03T05:17:09,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896, entries=150, sequenceid=593, filesize=12.0 K 2024-12-03T05:17:09,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/e401f5402bdb4e8e8d9d9af4a1c008e5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5 2024-12-03T05:17:09,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5, entries=150, sequenceid=593, filesize=12.0 K 2024-12-03T05:17:09,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6596af7287e3b5c33fe33f24d375a15b in 967ms, sequenceid=593, compaction requested=true 2024-12-03T05:17:09,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:09,877 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:09,878 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:09,878 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:09,878 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:09,879 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6769045f381b4c8d861772124f5a1e2d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=39.1 K 2024-12-03T05:17:09,879 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6769045f381b4c8d861772124f5a1e2d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733203027099 2024-12-03T05:17:09,880 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc75ca57c395485dbe54a08c19492e76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733203027823 2024-12-03T05:17:09,880 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3892cbb21d246809427b77af145e523, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028239 2024-12-03T05:17:09,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:09,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:09,887 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:09,888 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:09,888 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/efbd14e87481468b9933af8f00034795 is 50, key is test_row_0/A:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:09,890 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:09,890 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:09,890 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:09,890 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a105e13f6ebc4966b4523d7607b71c0d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.7 K 2024-12-03T05:17:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:09,891 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a105e13f6ebc4966b4523d7607b71c0d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733203027099 2024-12-03T05:17:09,891 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 480dad16a83b4ced932f057d11e6805e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733203027823 2024-12-03T05:17:09,891 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 159872d0fd6e4176825bf2a60148f896, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028889 2024-12-03T05:17:09,928 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:09,929 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/df55f9188d78480bafe16eabcc819e3e is 50, key is test_row_0/B:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:09,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742363_1539 (size=13119) 2024-12-03T05:17:09,937 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/efbd14e87481468b9933af8f00034795 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/efbd14e87481468b9933af8f00034795 2024-12-03T05:17:09,941 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into efbd14e87481468b9933af8f00034795(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:09,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:09,941 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203029877; duration=0sec 2024-12-03T05:17:09,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:09,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:09,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:17:09,942 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:17:09,942 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:09,942 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:09,942 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d654566568fd47d391ad958d359fd32c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=48.7 K 2024-12-03T05:17:09,943 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d654566568fd47d391ad958d359fd32c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733203025915 2024-12-03T05:17:09,943 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ea54a66c6a41d0ab3a03b992eb6aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733203027099 2024-12-03T05:17:09,943 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1275ea33ce42bebf9db12d2febcdf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733203027823 2024-12-03T05:17:09,943 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e401f5402bdb4e8e8d9d9af4a1c008e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028889 2024-12-03T05:17:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742364_1540 (size=13119) 2024-12-03T05:17:09,974 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:09,975 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc is 50, key is test_row_0/C:col10/1733203028905/Put/seqid=0 2024-12-03T05:17:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:17:10,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:10,011 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:10,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742365_1541 (size=13085) 2024-12-03T05:17:10,025 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc 2024-12-03T05:17:10,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/6d9e6932b26c452da8b3e136ee326609 is 50, key is test_row_0/A:col10/1733203029079/Put/seqid=0 2024-12-03T05:17:10,031 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into ef2186ff35aa4aaf8cd09ff3c72b3bdc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:10,031 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=12, startTime=1733203029891; duration=0sec 2024-12-03T05:17:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:10,031 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:10,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742366_1542 (size=12301) 2024-12-03T05:17:10,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203090089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203090211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203090241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203090241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203090250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203090260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:17:10,367 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/df55f9188d78480bafe16eabcc819e3e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/df55f9188d78480bafe16eabcc819e3e 2024-12-03T05:17:10,372 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into df55f9188d78480bafe16eabcc819e3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:10,372 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:10,372 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203029887; duration=0sec 2024-12-03T05:17:10,372 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:10,372 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203090432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,448 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/6d9e6932b26c452da8b3e136ee326609 2024-12-03T05:17:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/148bf4fa972f4b849a846677096b935f is 50, key is test_row_0/B:col10/1733203029079/Put/seqid=0 2024-12-03T05:17:10,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742367_1543 (size=12301) 2024-12-03T05:17:10,500 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/148bf4fa972f4b849a846677096b935f 2024-12-03T05:17:10,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5090319e885e4c3892668bc8b18318bc is 50, key is test_row_0/C:col10/1733203029079/Put/seqid=0 2024-12-03T05:17:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742368_1544 (size=12301) 2024-12-03T05:17:10,577 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5090319e885e4c3892668bc8b18318bc 2024-12-03T05:17:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/6d9e6932b26c452da8b3e136ee326609 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609 2024-12-03T05:17:10,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609, entries=150, sequenceid=628, filesize=12.0 K 2024-12-03T05:17:10,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/148bf4fa972f4b849a846677096b935f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f 2024-12-03T05:17:10,619 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f, entries=150, sequenceid=628, filesize=12.0 K 2024-12-03T05:17:10,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/5090319e885e4c3892668bc8b18318bc as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc 2024-12-03T05:17:10,624 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc, entries=150, sequenceid=628, filesize=12.0 K 2024-12-03T05:17:10,631 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 6596af7287e3b5c33fe33f24d375a15b in 619ms, sequenceid=628, compaction requested=false 2024-12-03T05:17:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T05:17:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T05:17:10,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T05:17:10,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 937 msec 2024-12-03T05:17:10,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 957 msec 2024-12-03T05:17:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:10,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:17:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:10,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f52189cb7bc64569a30ffe76a285eb0d is 50, key is test_row_0/A:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:10,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742369_1545 (size=14741) 2024-12-03T05:17:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T05:17:10,831 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:10,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T05:17:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:10,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:10,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:10,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203090883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:10,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:17:10,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:10,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:10,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:10,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:10,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203090992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:17:11,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:11,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f52189cb7bc64569a30ffe76a285eb0d 2024-12-03T05:17:11,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/ea06f7bb303b41d1b5f267cf89f9cd9c is 50, key is test_row_0/B:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:11,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742370_1546 (size=12301) 2024-12-03T05:17:11,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/ea06f7bb303b41d1b5f267cf89f9cd9c 2024-12-03T05:17:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203091210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d3955e6aec134316aa5f189dd8789c63 is 50, key is test_row_0/C:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:11,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742371_1547 (size=12301) 2024-12-03T05:17:11,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d3955e6aec134316aa5f189dd8789c63 2024-12-03T05:17:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f52189cb7bc64569a30ffe76a285eb0d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d 2024-12-03T05:17:11,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d, entries=200, sequenceid=642, filesize=14.4 K 2024-12-03T05:17:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203091260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203091261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,267 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,268 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/ea06f7bb303b41d1b5f267cf89f9cd9c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c 2024-12-03T05:17:11,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c, entries=150, sequenceid=642, filesize=12.0 K 2024-12-03T05:17:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/d3955e6aec134316aa5f189dd8789c63 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63 2024-12-03T05:17:11,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203091280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203091280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,284 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,285 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 6596af7287e3b5c33fe33f24d375a15b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:17:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63, entries=150, sequenceid=642, filesize=12.0 K 2024-12-03T05:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:11,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6596af7287e3b5c33fe33f24d375a15b in 550ms, sequenceid=642, compaction requested=true 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:11,298 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:11,298 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:11,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:11,301 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:11,301 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:11,301 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,301 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/efbd14e87481468b9933af8f00034795, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=39.2 K 2024-12-03T05:17:11,301 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:11,301 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:11,301 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,301 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/df55f9188d78480bafe16eabcc819e3e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.8 K 2024-12-03T05:17:11,302 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting df55f9188d78480bafe16eabcc819e3e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028889 2024-12-03T05:17:11,303 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbd14e87481468b9933af8f00034795, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028889 2024-12-03T05:17:11,303 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 148bf4fa972f4b849a846677096b935f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1733203029063 2024-12-03T05:17:11,303 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ea06f7bb303b41d1b5f267cf89f9cd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:11,304 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d9e6932b26c452da8b3e136ee326609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1733203029063 2024-12-03T05:17:11,304 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52189cb7bc64569a30ffe76a285eb0d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:11,312 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:11,313 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/28be58938df14d75a10517b4e58afb69 is 50, key is test_row_0/B:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:11,329 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#465 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:11,330 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e is 50, key is test_row_0/A:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742372_1548 (size=13221) 2024-12-03T05:17:11,354 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/28be58938df14d75a10517b4e58afb69 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/28be58938df14d75a10517b4e58afb69 2024-12-03T05:17:11,361 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into 28be58938df14d75a10517b4e58afb69(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:11,361 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:11,362 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203031298; duration=0sec 2024-12-03T05:17:11,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:11,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:11,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:11,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:11,362 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:11,363 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,363 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.8 K 2024-12-03T05:17:11,363 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2186ff35aa4aaf8cd09ff3c72b3bdc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733203028889 2024-12-03T05:17:11,363 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5090319e885e4c3892668bc8b18318bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1733203029063 2024-12-03T05:17:11,363 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting d3955e6aec134316aa5f189dd8789c63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:11,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742373_1549 (size=13221) 2024-12-03T05:17:11,382 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:11,382 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/c8dd9689f41644868b8e4c08e9acd28e is 50, key is test_row_0/C:col10/1733203030072/Put/seqid=0 2024-12-03T05:17:11,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742374_1550 (size=13187) 2024-12-03T05:17:11,432 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/c8dd9689f41644868b8e4c08e9acd28e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8dd9689f41644868b8e4c08e9acd28e 2024-12-03T05:17:11,436 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into c8dd9689f41644868b8e4c08e9acd28e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:11,436 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:11,436 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=13, startTime=1733203031298; duration=0sec 2024-12-03T05:17:11,437 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:11,437 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:11,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T05:17:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:11,449 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f97cbd8d85aa498999c5ed1e58f95804 is 50, key is test_row_0/A:col10/1733203030881/Put/seqid=0 2024-12-03T05:17:11,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742375_1551 (size=12301) 2024-12-03T05:17:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:11,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203091585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203091700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,786 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e 2024-12-03T05:17:11,801 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into 5daf6e4d4e0e4bb6bb1f18a39e3af00e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:11,801 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:11,801 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203031298; duration=0sec 2024-12-03T05:17:11,801 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:11,801 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:11,896 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f97cbd8d85aa498999c5ed1e58f95804 2024-12-03T05:17:11,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/da94f1843f5640d88b92bfd3c79e174d is 50, key is test_row_0/B:col10/1733203030881/Put/seqid=0 2024-12-03T05:17:11,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742376_1552 (size=12301) 2024-12-03T05:17:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/da94f1843f5640d88b92bfd3c79e174d 2024-12-03T05:17:11,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203091911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/76948b9b7ebc4d5889ac7d5da839e2a7 is 50, key is test_row_0/C:col10/1733203030881/Put/seqid=0 2024-12-03T05:17:11,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742377_1553 (size=12301) 2024-12-03T05:17:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:12,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203092224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:12,340 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/76948b9b7ebc4d5889ac7d5da839e2a7 2024-12-03T05:17:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/f97cbd8d85aa498999c5ed1e58f95804 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804 2024-12-03T05:17:12,369 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804, entries=150, sequenceid=664, filesize=12.0 K 2024-12-03T05:17:12,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/da94f1843f5640d88b92bfd3c79e174d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d 2024-12-03T05:17:12,390 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d, entries=150, sequenceid=664, filesize=12.0 K 2024-12-03T05:17:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/76948b9b7ebc4d5889ac7d5da839e2a7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7 2024-12-03T05:17:12,394 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7, entries=150, sequenceid=664, filesize=12.0 K 2024-12-03T05:17:12,398 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6596af7287e3b5c33fe33f24d375a15b in 949ms, sequenceid=664, compaction requested=false 2024-12-03T05:17:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T05:17:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T05:17:12,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T05:17:12,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-12-03T05:17:12,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5900 sec 2024-12-03T05:17:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:12,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:17:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:12,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4231b85489154b7cbbdbd44f5cb4f01b is 50, key is test_row_0/A:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:12,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742378_1554 (size=14741) 2024-12-03T05:17:12,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4231b85489154b7cbbdbd44f5cb4f01b 2024-12-03T05:17:12,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f1d20dbdd90a4447a0f279f974750654 is 50, key is test_row_0/B:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:12,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742379_1555 (size=12301) 2024-12-03T05:17:12,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f1d20dbdd90a4447a0f279f974750654 2024-12-03T05:17:12,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/07f318b156f94f3a91b1c08d68716ec9 is 50, key is test_row_0/C:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:12,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742380_1556 (size=12301) 2024-12-03T05:17:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203092914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T05:17:12,993 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:12,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-03T05:17:13,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:13,011 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:13,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203093031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:13,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T05:17:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/07f318b156f94f3a91b1c08d68716ec9 2024-12-03T05:17:13,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/4231b85489154b7cbbdbd44f5cb4f01b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b 2024-12-03T05:17:13,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b, entries=200, sequenceid=678, filesize=14.4 K 2024-12-03T05:17:13,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/f1d20dbdd90a4447a0f279f974750654 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654 2024-12-03T05:17:13,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654, entries=150, sequenceid=678, filesize=12.0 K 2024-12-03T05:17:13,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/07f318b156f94f3a91b1c08d68716ec9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9 2024-12-03T05:17:13,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9, entries=150, sequenceid=678, filesize=12.0 K 2024-12-03T05:17:13,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6596af7287e3b5c33fe33f24d375a15b in 504ms, sequenceid=678, compaction requested=true 2024-12-03T05:17:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:13,246 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:13,247 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6596af7287e3b5c33fe33f24d375a15b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:13,247 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:13,248 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/A is initiating minor compaction (all files) 2024-12-03T05:17:13,248 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/A in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,248 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=39.3 K 2024-12-03T05:17:13,249 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5daf6e4d4e0e4bb6bb1f18a39e3af00e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:13,249 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97cbd8d85aa498999c5ed1e58f95804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733203030827 2024-12-03T05:17:13,249 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:13,250 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4231b85489154b7cbbdbd44f5cb4f01b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733203031563 2024-12-03T05:17:13,250 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/B is initiating minor compaction (all files) 2024-12-03T05:17:13,250 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/B in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,250 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/28be58938df14d75a10517b4e58afb69, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.9 K 2024-12-03T05:17:13,253 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 28be58938df14d75a10517b4e58afb69, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:13,253 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting da94f1843f5640d88b92bfd3c79e174d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733203030827 2024-12-03T05:17:13,254 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d20dbdd90a4447a0f279f974750654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733203031563 2024-12-03T05:17:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:13,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:17:13,264 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#A#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,265 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/bce25494557446fda22f711b69abf68d is 50, key is test_row_0/A:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:13,274 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#B#compaction#474 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:13,274 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/100c2041babd4c8d9ec6e0da8cf3453a is 50, key is test_row_0/B:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:13,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/69b97dad1ae949d28cc5e017c4d35cf5 is 50, key is test_row_0/A:col10/1733203033259/Put/seqid=0 2024-12-03T05:17:13,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742381_1557 (size=13323) 2024-12-03T05:17:13,285 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/bce25494557446fda22f711b69abf68d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bce25494557446fda22f711b69abf68d 2024-12-03T05:17:13,294 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/A of 6596af7287e3b5c33fe33f24d375a15b into bce25494557446fda22f711b69abf68d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:13,294 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:13,294 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/A, priority=13, startTime=1733203033246; duration=0sec 2024-12-03T05:17:13,295 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:13,295 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:A 2024-12-03T05:17:13,295 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:13,296 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:13,296 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 6596af7287e3b5c33fe33f24d375a15b/C is initiating minor compaction (all files) 2024-12-03T05:17:13,296 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6596af7287e3b5c33fe33f24d375a15b/C in TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,296 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8dd9689f41644868b8e4c08e9acd28e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp, totalSize=36.9 K 2024-12-03T05:17:13,299 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8dd9689f41644868b8e4c08e9acd28e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1733203030072 2024-12-03T05:17:13,300 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76948b9b7ebc4d5889ac7d5da839e2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733203030827 2024-12-03T05:17:13,300 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f318b156f94f3a91b1c08d68716ec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733203031563 2024-12-03T05:17:13,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742382_1558 (size=14741) 2024-12-03T05:17:13,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/69b97dad1ae949d28cc5e017c4d35cf5 2024-12-03T05:17:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:13,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T05:17:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203093309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203093310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203093312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203093314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742383_1559 (size=13323) 2024-12-03T05:17:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203093327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,350 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/100c2041babd4c8d9ec6e0da8cf3453a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/100c2041babd4c8d9ec6e0da8cf3453a 2024-12-03T05:17:13,357 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6596af7287e3b5c33fe33f24d375a15b#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:13,357 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/46e1ab0ef0b24dbdb125ec675436d821 is 50, key is test_row_0/C:col10/1733203031563/Put/seqid=0 2024-12-03T05:17:13,360 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/B of 6596af7287e3b5c33fe33f24d375a15b into 100c2041babd4c8d9ec6e0da8cf3453a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:13,360 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:13,360 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/B, priority=13, startTime=1733203033247; duration=0sec 2024-12-03T05:17:13,360 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:13,360 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:B 2024-12-03T05:17:13,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/d55acf6b1f22432984d6628c8eb40bb1 is 50, key is test_row_0/B:col10/1733203033259/Put/seqid=0 2024-12-03T05:17:13,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742384_1560 (size=13289) 2024-12-03T05:17:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742385_1561 (size=12301) 2024-12-03T05:17:13,389 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/46e1ab0ef0b24dbdb125ec675436d821 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/46e1ab0ef0b24dbdb125ec675436d821 2024-12-03T05:17:13,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/d55acf6b1f22432984d6628c8eb40bb1 2024-12-03T05:17:13,395 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6596af7287e3b5c33fe33f24d375a15b/C of 6596af7287e3b5c33fe33f24d375a15b into 46e1ab0ef0b24dbdb125ec675436d821(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:13,395 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:13,395 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., storeName=6596af7287e3b5c33fe33f24d375a15b/C, priority=13, startTime=1733203033247; duration=0sec 2024-12-03T05:17:13,395 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:13,395 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6596af7287e3b5c33fe33f24d375a15b:C 2024-12-03T05:17:13,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/82df239ba28949cd8d5807564c033589 is 50, key is test_row_0/C:col10/1733203033259/Put/seqid=0 2024-12-03T05:17:13,409 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-12-03T05:17:13,409 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,409 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,409 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,410 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-12-03T05:17:13,410 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,410 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,410 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,413 INFO [Thread-2104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2104. 2024-12-03T05:17:13,413 DEBUG [Thread-2104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,413 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,413 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,419 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-12-03T05:17:13,419 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,419 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,419 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,425 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-12-03T05:17:13,425 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,425 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,425 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742386_1562 (size=12301) 2024-12-03T05:17:13,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/82df239ba28949cd8d5807564c033589 2024-12-03T05:17:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33678 deadline: 1733203093431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1733203093430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33638 deadline: 1733203093431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33652 deadline: 1733203093431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/69b97dad1ae949d28cc5e017c4d35cf5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/69b97dad1ae949d28cc5e017c4d35cf5 2024-12-03T05:17:13,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/69b97dad1ae949d28cc5e017c4d35cf5, entries=200, sequenceid=698, filesize=14.4 K 2024-12-03T05:17:13,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/d55acf6b1f22432984d6628c8eb40bb1 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d55acf6b1f22432984d6628c8eb40bb1 2024-12-03T05:17:13,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33724 deadline: 1733203093450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 , the old value is region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., hostname=e3c6570475b6,37863,1733202887099, seqNum=2 because the exception is null or not the one we care about 2024-12-03T05:17:13,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d55acf6b1f22432984d6628c8eb40bb1, entries=150, sequenceid=698, filesize=12.0 K 2024-12-03T05:17:13,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/82df239ba28949cd8d5807564c033589 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/82df239ba28949cd8d5807564c033589 2024-12-03T05:17:13,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/82df239ba28949cd8d5807564c033589, entries=150, sequenceid=698, filesize=12.0 K 2024-12-03T05:17:13,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6596af7287e3b5c33fe33f24d375a15b in 196ms, sequenceid=698, compaction requested=false 2024-12-03T05:17:13,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:13,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:13,474 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:13,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5d8d1128db074feead1fabe3186c88a4 is 50, key is test_row_0/A:col10/1733203033309/Put/seqid=0 2024-12-03T05:17:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742387_1563 (size=9857) 2024-12-03T05:17:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:13,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. as already flushing 2024-12-03T05:17:13,642 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-12-03T05:17:13,642 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-12-03T05:17:13,642 INFO [Thread-2093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2093. 2024-12-03T05:17:13,642 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,642 DEBUG [Thread-2093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,642 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,642 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,642 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,642 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,642 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,642 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,643 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,644 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-12-03T05:17:13,644 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,644 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,644 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,662 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-12-03T05:17:13,663 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:13,663 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,663 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:13,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:13,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=713 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5d8d1128db074feead1fabe3186c88a4 2024-12-03T05:17:13,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a6eff92f8698431a97c4deec30167381 is 50, key is test_row_0/B:col10/1733203033309/Put/seqid=0 2024-12-03T05:17:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742388_1564 (size=9857) 2024-12-03T05:17:13,924 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=713 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a6eff92f8698431a97c4deec30167381 2024-12-03T05:17:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/78e517b3d5154fb9bf7aa477e3dcaca8 is 50, key is test_row_0/C:col10/1733203033309/Put/seqid=0 2024-12-03T05:17:13,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742389_1565 (size=9857) 2024-12-03T05:17:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:14,343 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=713 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/78e517b3d5154fb9bf7aa477e3dcaca8 2024-12-03T05:17:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/5d8d1128db074feead1fabe3186c88a4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5d8d1128db074feead1fabe3186c88a4 2024-12-03T05:17:14,368 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5d8d1128db074feead1fabe3186c88a4, entries=100, sequenceid=713, filesize=9.6 K 2024-12-03T05:17:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/a6eff92f8698431a97c4deec30167381 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a6eff92f8698431a97c4deec30167381 2024-12-03T05:17:14,376 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a6eff92f8698431a97c4deec30167381, entries=100, sequenceid=713, filesize=9.6 K 2024-12-03T05:17:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/78e517b3d5154fb9bf7aa477e3dcaca8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/78e517b3d5154fb9bf7aa477e3dcaca8 2024-12-03T05:17:14,394 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/78e517b3d5154fb9bf7aa477e3dcaca8, entries=100, sequenceid=713, filesize=9.6 K 2024-12-03T05:17:14,395 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 6596af7287e3b5c33fe33f24d375a15b in 921ms, sequenceid=713, compaction requested=true 2024-12-03T05:17:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6596af7287e3b5c33fe33f24d375a15b: 2024-12-03T05:17:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-03T05:17:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-03T05:17:14,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T05:17:14,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3860 sec 2024-12-03T05:17:14,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4060 sec 2024-12-03T05:17:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T05:17:15,161 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 276 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1039 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3116 rows 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1046 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3137 rows 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3072 rows 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1027 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3080 rows 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1030 2024-12-03T05:17:15,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3089 rows 2024-12-03T05:17:15,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:17:15,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:17:15,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:15,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:15,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:15,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:17:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:15,170 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203035170"}]},"ts":"1733203035170"} 2024-12-03T05:17:15,172 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:17:15,172 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:17:15,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:17:15,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, UNASSIGN}] 2024-12-03T05:17:15,177 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, UNASSIGN 2024-12-03T05:17:15,178 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6596af7287e3b5c33fe33f24d375a15b, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:15,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, UNASSIGN because future has completed 2024-12-03T05:17:15,180 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:17:15,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:17:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:15,333 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:15,333 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:17:15,333 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 6596af7287e3b5c33fe33f24d375a15b, disabling compactions & flushes 2024-12-03T05:17:15,334 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. after waiting 0 ms 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:15,334 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 6596af7287e3b5c33fe33f24d375a15b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=A 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=B 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6596af7287e3b5c33fe33f24d375a15b, store=C 2024-12-03T05:17:15,334 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:15,337 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/7a7e969d40794624a804f58baf8295d7 is 50, key is test_row_0/A:col10/1733203033642/Put/seqid=0 2024-12-03T05:17:15,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742390_1566 (size=12301) 2024-12-03T05:17:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:15,483 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:17:15,770 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/7a7e969d40794624a804f58baf8295d7 2024-12-03T05:17:15,778 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3c1051f883aa40c2b29c6611f464cfc3 is 50, key is test_row_0/B:col10/1733203033642/Put/seqid=0 2024-12-03T05:17:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:15,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742391_1567 (size=12301) 2024-12-03T05:17:15,810 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3c1051f883aa40c2b29c6611f464cfc3 2024-12-03T05:17:15,818 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/de974d90e64a4d83ba6f63de3eb2178c is 50, key is test_row_0/C:col10/1733203033642/Put/seqid=0 2024-12-03T05:17:15,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742392_1568 (size=12301) 2024-12-03T05:17:16,234 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/de974d90e64a4d83ba6f63de3eb2178c 2024-12-03T05:17:16,241 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/A/7a7e969d40794624a804f58baf8295d7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7a7e969d40794624a804f58baf8295d7 2024-12-03T05:17:16,245 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7a7e969d40794624a804f58baf8295d7, entries=150, sequenceid=721, filesize=12.0 K 2024-12-03T05:17:16,247 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/B/3c1051f883aa40c2b29c6611f464cfc3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3c1051f883aa40c2b29c6611f464cfc3 2024-12-03T05:17:16,254 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3c1051f883aa40c2b29c6611f464cfc3, entries=150, sequenceid=721, filesize=12.0 K 2024-12-03T05:17:16,256 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/.tmp/C/de974d90e64a4d83ba6f63de3eb2178c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/de974d90e64a4d83ba6f63de3eb2178c 2024-12-03T05:17:16,270 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/de974d90e64a4d83ba6f63de3eb2178c, entries=150, sequenceid=721, filesize=12.0 K 2024-12-03T05:17:16,271 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6596af7287e3b5c33fe33f24d375a15b in 937ms, sequenceid=721, compaction requested=true 2024-12-03T05:17:16,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/cb4d234bec274165916a97ef5deb204f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/37fe445d817a4b06b2f28de7a2b3a803, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/63f975f349284dabbd0d20f6ca18cab5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/1ae88f177e5a4dafa4c126476a4e6a55, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7eaa4d8f747e4c20a87a46d80fd62027, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6769045f381b4c8d861772124f5a1e2d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/efbd14e87481468b9933af8f00034795, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b] to archive 2024-12-03T05:17:16,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:16,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/fbad58e7916c460ba66f742a15e91b19 2024-12-03T05:17:16,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/d474537d32f147c1a6967e969d41c27f 2024-12-03T05:17:16,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5825f809fa0e4991a932c2fe48478465 2024-12-03T05:17:16,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a22c02e098804ee985a2a9da61a8a979 2024-12-03T05:17:16,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/cb4d234bec274165916a97ef5deb204f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/cb4d234bec274165916a97ef5deb204f 2024-12-03T05:17:16,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/224321f9682648beb007e677eea8b143 2024-12-03T05:17:16,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4e4fd79e30e04bf4b8404e5e4d89bf2e 2024-12-03T05:17:16,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/79492ddf90cc4b20ae99183a1dfb1377 2024-12-03T05:17:16,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/176461fbb5e446288cb04e4583d48fb5 2024-12-03T05:17:16,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/37fe445d817a4b06b2f28de7a2b3a803 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/37fe445d817a4b06b2f28de7a2b3a803 2024-12-03T05:17:16,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2b925ebbc82047d29fb7da0556ddccd0 2024-12-03T05:17:16,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/ed2b65c587f240ae8fa4251c15eb7cd9 2024-12-03T05:17:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:16,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/63f975f349284dabbd0d20f6ca18cab5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/63f975f349284dabbd0d20f6ca18cab5 2024-12-03T05:17:16,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/032bf0627f554ae7b1fa8516031a7c1a 2024-12-03T05:17:16,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/1ae88f177e5a4dafa4c126476a4e6a55 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/1ae88f177e5a4dafa4c126476a4e6a55 2024-12-03T05:17:16,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/2a43c399b0984965afdf2b40eaba246e 2024-12-03T05:17:16,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/82144988588b40fdb704e02b8d378e72 2024-12-03T05:17:16,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/8a9642680b7144efb4683effa80c3a7d 2024-12-03T05:17:16,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7eaa4d8f747e4c20a87a46d80fd62027 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7eaa4d8f747e4c20a87a46d80fd62027 2024-12-03T05:17:16,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f9112dee62d942d69bfbdb0cef23026f 2024-12-03T05:17:16,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6769045f381b4c8d861772124f5a1e2d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6769045f381b4c8d861772124f5a1e2d 2024-12-03T05:17:16,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/e9da728427e34c22a3ea6f1a90028cc6 2024-12-03T05:17:16,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bc75ca57c395485dbe54a08c19492e76 2024-12-03T05:17:16,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/a3892cbb21d246809427b77af145e523 2024-12-03T05:17:16,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/efbd14e87481468b9933af8f00034795 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/efbd14e87481468b9933af8f00034795 2024-12-03T05:17:16,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/6d9e6932b26c452da8b3e136ee326609 2024-12-03T05:17:16,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f52189cb7bc64569a30ffe76a285eb0d 2024-12-03T05:17:16,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5daf6e4d4e0e4bb6bb1f18a39e3af00e 2024-12-03T05:17:16,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/f97cbd8d85aa498999c5ed1e58f95804 2024-12-03T05:17:16,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/4231b85489154b7cbbdbd44f5cb4f01b 2024-12-03T05:17:16,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e362ecaede334ff895d82fe3de30d784, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e6022eb946bb4d0181eda9cf1f2cbbb7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a9e1115a4eb9454281b95a4dab695b2f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/2b0fbbbf4db24256bc4bf797d8f02eac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d82d44184e284460b1d76ba52f7ce5eb, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a105e13f6ebc4966b4523d7607b71c0d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/df55f9188d78480bafe16eabcc819e3e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/28be58938df14d75a10517b4e58afb69, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654] to archive 2024-12-03T05:17:16,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:16,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/329a580cdf6f460fbf383b2a7b42e513 2024-12-03T05:17:16,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/bb5fe59da94c45f48c19efe0984adec5 2024-12-03T05:17:16,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/26b53300b5084ba983012ce0cca2b26a 2024-12-03T05:17:16,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e362ecaede334ff895d82fe3de30d784 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e362ecaede334ff895d82fe3de30d784 2024-12-03T05:17:16,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/5839b0ae8ab744eda62a887ffe9dc95c 2024-12-03T05:17:16,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/18dc88ef7b7e4af3b3c8bd1b42195866 2024-12-03T05:17:16,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/335ad8839aa44747bb50a2724b02f53d 2024-12-03T05:17:16,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f234c691b104461a9ccb1b7d715634bd 2024-12-03T05:17:16,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e6022eb946bb4d0181eda9cf1f2cbbb7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/e6022eb946bb4d0181eda9cf1f2cbbb7 2024-12-03T05:17:16,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/04c0c93ea4da4688948b61b5cc500764 2024-12-03T05:17:16,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3a9cc336134e4198a716da8824a63c7c 2024-12-03T05:17:16,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a9e1115a4eb9454281b95a4dab695b2f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a9e1115a4eb9454281b95a4dab695b2f 2024-12-03T05:17:16,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/76811f12e960494790425e36d8f1de7d 2024-12-03T05:17:16,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/14688510733e466e9e6953a7af061030 2024-12-03T05:17:16,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/2b0fbbbf4db24256bc4bf797d8f02eac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/2b0fbbbf4db24256bc4bf797d8f02eac 2024-12-03T05:17:16,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/fa7d6150e092459cb387b59d58b0e91e 2024-12-03T05:17:16,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3f57abbe7c9b48f7b7265194beda1c9c 2024-12-03T05:17:16,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d82d44184e284460b1d76ba52f7ce5eb to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d82d44184e284460b1d76ba52f7ce5eb 2024-12-03T05:17:16,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/35796910cec54f2280a15463e2cd8ca1 2024-12-03T05:17:16,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/7dcad4ee0d7649afbce28ec35957fba2 2024-12-03T05:17:16,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a105e13f6ebc4966b4523d7607b71c0d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a105e13f6ebc4966b4523d7607b71c0d 2024-12-03T05:17:16,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/af010a405d514cb2a5ddfca7fb101a12 2024-12-03T05:17:16,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/480dad16a83b4ced932f057d11e6805e 2024-12-03T05:17:16,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/df55f9188d78480bafe16eabcc819e3e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/df55f9188d78480bafe16eabcc819e3e 2024-12-03T05:17:16,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/159872d0fd6e4176825bf2a60148f896 2024-12-03T05:17:16,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/148bf4fa972f4b849a846677096b935f 2024-12-03T05:17:16,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/28be58938df14d75a10517b4e58afb69 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/28be58938df14d75a10517b4e58afb69 2024-12-03T05:17:16,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/ea06f7bb303b41d1b5f267cf89f9cd9c 2024-12-03T05:17:16,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/da94f1843f5640d88b92bfd3c79e174d 2024-12-03T05:17:16,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/f1d20dbdd90a4447a0f279f974750654 2024-12-03T05:17:16,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/662de95187534d59aa1e6a867be6edaf, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/bbf1d47e11ba4801944463e13dee482e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/67a8bc975a51449e987b2d6e1ba99966, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/35594ea7b40641a28319ee529414c250, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d654566568fd47d391ad958d359fd32c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8dd9689f41644868b8e4c08e9acd28e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9] to archive 2024-12-03T05:17:16,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:16,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/87c9fb49d103489d876e1737c2cc5556 2024-12-03T05:17:16,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d7a7efc1fbd44776b513f4d0ae2cccb3 2024-12-03T05:17:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5490d01057f342d09c5599af577d5016 2024-12-03T05:17:16,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/662de95187534d59aa1e6a867be6edaf to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/662de95187534d59aa1e6a867be6edaf 2024-12-03T05:17:16,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6c42aa2b936b4a2c86d1440a2f51446a 2024-12-03T05:17:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/7ad8a28dd3ba473eac844aa954501737 2024-12-03T05:17:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/6b6973b2cd994ff99539a65c28bf862e 2024-12-03T05:17:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/fce3c7514eba4acabcd59bde0cd7755a 2024-12-03T05:17:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/bbf1d47e11ba4801944463e13dee482e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/bbf1d47e11ba4801944463e13dee482e 2024-12-03T05:17:16,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8bdd01f678a4999bf632aef77c9cadd 2024-12-03T05:17:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/05b0f19b9b9744e5a2b93fdd439b3ed5 2024-12-03T05:17:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/30426846a90b47deb69f7fe82648e695 2024-12-03T05:17:16,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/67a8bc975a51449e987b2d6e1ba99966 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/67a8bc975a51449e987b2d6e1ba99966 2024-12-03T05:17:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/deab131d0a5c4f09aa1bcbd73fb3ab14 2024-12-03T05:17:16,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/556fc563db704f0eb8bf11823ce4324a 2024-12-03T05:17:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/35594ea7b40641a28319ee529414c250 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/35594ea7b40641a28319ee529414c250 2024-12-03T05:17:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/23b8e11ed87644a4a5fd791f8557c574 2024-12-03T05:17:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3a3e45c0713c4c45bda9d17257f48735 2024-12-03T05:17:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d654566568fd47d391ad958d359fd32c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d654566568fd47d391ad958d359fd32c 2024-12-03T05:17:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5ef1a531fd3c4c24afcf19ebb5f3e0c7 2024-12-03T05:17:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/40ea54a66c6a41d0ab3a03b992eb6aac 2024-12-03T05:17:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/3d1275ea33ce42bebf9db12d2febcdf5 2024-12-03T05:17:16,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/ef2186ff35aa4aaf8cd09ff3c72b3bdc 2024-12-03T05:17:16,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/e401f5402bdb4e8e8d9d9af4a1c008e5 2024-12-03T05:17:16,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/5090319e885e4c3892668bc8b18318bc 2024-12-03T05:17:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8dd9689f41644868b8e4c08e9acd28e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/c8dd9689f41644868b8e4c08e9acd28e 2024-12-03T05:17:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/d3955e6aec134316aa5f189dd8789c63 2024-12-03T05:17:16,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/76948b9b7ebc4d5889ac7d5da839e2a7 2024-12-03T05:17:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/07f318b156f94f3a91b1c08d68716ec9 2024-12-03T05:17:16,452 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/recovered.edits/724.seqid, newMaxSeqId=724, maxSeqId=1 2024-12-03T05:17:16,453 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. 2024-12-03T05:17:16,453 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 6596af7287e3b5c33fe33f24d375a15b: Waiting for close lock at 1733203035333Running coprocessor pre-close hooks at 1733203035333Disabling compacts and flushes for region at 1733203035333Disabling writes for close at 1733203035334 (+1 ms)Obtaining lock to block concurrent updates at 1733203035334Preparing flush snapshotting stores in 6596af7287e3b5c33fe33f24d375a15b at 1733203035334Finished memstore snapshotting TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733203035334Flushing stores of TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b. at 1733203035334Flushing 6596af7287e3b5c33fe33f24d375a15b/A: creating writer at 1733203035334Flushing 6596af7287e3b5c33fe33f24d375a15b/A: appending metadata at 1733203035337 (+3 ms)Flushing 6596af7287e3b5c33fe33f24d375a15b/A: closing flushed file at 1733203035337Flushing 6596af7287e3b5c33fe33f24d375a15b/B: creating writer at 1733203035774 (+437 ms)Flushing 6596af7287e3b5c33fe33f24d375a15b/B: appending metadata at 1733203035777 (+3 ms)Flushing 6596af7287e3b5c33fe33f24d375a15b/B: closing flushed file at 1733203035777Flushing 6596af7287e3b5c33fe33f24d375a15b/C: creating writer at 1733203035815 (+38 ms)Flushing 6596af7287e3b5c33fe33f24d375a15b/C: appending metadata at 1733203035818 (+3 ms)Flushing 6596af7287e3b5c33fe33f24d375a15b/C: closing flushed file at 1733203035818Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f81fab8: reopening flushed file at 1733203036238 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@470f140b: reopening flushed file at 1733203036246 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e189465: reopening flushed file at 1733203036254 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6596af7287e3b5c33fe33f24d375a15b in 937ms, sequenceid=721, compaction requested=true at 1733203036271 (+17 ms)Writing region close event to WAL at 1733203036450 (+179 ms)Running coprocessor post-close hooks at 1733203036453 (+3 ms)Closed at 1733203036453 2024-12-03T05:17:16,456 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:16,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6596af7287e3b5c33fe33f24d375a15b, regionState=CLOSED 2024-12-03T05:17:16,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:17:16,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T05:17:16,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 6596af7287e3b5c33fe33f24d375a15b, server=e3c6570475b6,37863,1733202887099 in 1.2790 sec 2024-12-03T05:17:16,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-03T05:17:16,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6596af7287e3b5c33fe33f24d375a15b, UNASSIGN in 1.2850 sec 2024-12-03T05:17:16,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T05:17:16,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2890 sec 2024-12-03T05:17:16,466 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203036466"}]},"ts":"1733203036466"} 2024-12-03T05:17:16,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:17:16,467 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:17:16,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3050 sec 2024-12-03T05:17:16,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T05:17:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T05:17:17,311 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:17,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:17:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,313 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,314 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T05:17:17,316 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:17,318 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/recovered.edits] 2024-12-03T05:17:17,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5d8d1128db074feead1fabe3186c88a4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/5d8d1128db074feead1fabe3186c88a4 2024-12-03T05:17:17,321 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/69b97dad1ae949d28cc5e017c4d35cf5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/69b97dad1ae949d28cc5e017c4d35cf5 2024-12-03T05:17:17,323 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7a7e969d40794624a804f58baf8295d7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/7a7e969d40794624a804f58baf8295d7 2024-12-03T05:17:17,324 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bce25494557446fda22f711b69abf68d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/A/bce25494557446fda22f711b69abf68d 2024-12-03T05:17:17,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/100c2041babd4c8d9ec6e0da8cf3453a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/100c2041babd4c8d9ec6e0da8cf3453a 2024-12-03T05:17:17,328 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3c1051f883aa40c2b29c6611f464cfc3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/3c1051f883aa40c2b29c6611f464cfc3 2024-12-03T05:17:17,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a6eff92f8698431a97c4deec30167381 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/a6eff92f8698431a97c4deec30167381 2024-12-03T05:17:17,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d55acf6b1f22432984d6628c8eb40bb1 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/B/d55acf6b1f22432984d6628c8eb40bb1 2024-12-03T05:17:17,338 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/46e1ab0ef0b24dbdb125ec675436d821 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/46e1ab0ef0b24dbdb125ec675436d821 2024-12-03T05:17:17,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/78e517b3d5154fb9bf7aa477e3dcaca8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/78e517b3d5154fb9bf7aa477e3dcaca8 2024-12-03T05:17:17,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/82df239ba28949cd8d5807564c033589 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/82df239ba28949cd8d5807564c033589 2024-12-03T05:17:17,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/de974d90e64a4d83ba6f63de3eb2178c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/C/de974d90e64a4d83ba6f63de3eb2178c 2024-12-03T05:17:17,344 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/recovered.edits/724.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b/recovered.edits/724.seqid 2024-12-03T05:17:17,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/6596af7287e3b5c33fe33f24d375a15b 2024-12-03T05:17:17,345 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:17:17,347 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,350 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:17:17,357 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:17:17,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:17:17,366 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733203037365"}]},"ts":"9223372036854775807"} 2024-12-03T05:17:17,367 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:17:17,367 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6596af7287e3b5c33fe33f24d375a15b, NAME => 'TestAcidGuarantees,,1733203011200.6596af7287e3b5c33fe33f24d375a15b.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:17:17,367 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:17:17,367 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733203037367"}]},"ts":"9223372036854775807"} 2024-12-03T05:17:17,369 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:17:17,370 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-12-03T05:17:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T05:17:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:17:17,420 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:17,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=773 (was 757) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2426 (was 3021) 2024-12-03T05:17:17,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=773, ProcessCount=11, AvailableMemoryMB=2426 2024-12-03T05:17:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:17:17,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:17:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:17,444 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:17:17,444 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:17,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-03T05:17:17,445 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:17:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T05:17:17,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742393_1569 (size=1061) 2024-12-03T05:17:17,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd 2024-12-03T05:17:17,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742394_1570 (size=53) 2024-12-03T05:17:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T05:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9376883f3ccddf1028e50b0de6a8fad3, disabling compactions & flushes 2024-12-03T05:17:17,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. after waiting 0 ms 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:17,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:17,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9376883f3ccddf1028e50b0de6a8fad3: Waiting for close lock at 1733203037860Disabling compacts and flushes for region at 1733203037860Disabling writes for close at 1733203037860Writing region close event to WAL at 1733203037860Closed at 1733203037860 2024-12-03T05:17:17,861 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:17:17,861 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733203037861"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733203037861"}]},"ts":"1733203037861"} 2024-12-03T05:17:17,863 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:17:17,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:17:17,864 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203037864"}]},"ts":"1733203037864"} 2024-12-03T05:17:17,866 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T05:17:17,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, ASSIGN}] 2024-12-03T05:17:17,868 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, ASSIGN 2024-12-03T05:17:17,868 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, ASSIGN; state=OFFLINE, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=false 2024-12-03T05:17:18,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:18,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, ASSIGN because future has completed 2024-12-03T05:17:18,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T05:17:18,176 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:18,177 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:17:18,177 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,177 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:17:18,177 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,178 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,179 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,181 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:18,181 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName A 2024-12-03T05:17:18,181 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:18,181 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:18,181 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,182 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:18,183 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName B 2024-12-03T05:17:18,183 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:18,183 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:18,183 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,184 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:18,184 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName C 2024-12-03T05:17:18,184 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:18,184 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:18,185 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:18,185 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,185 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,185 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,186 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,186 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,187 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:17:18,187 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,189 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:17:18,189 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 9376883f3ccddf1028e50b0de6a8fad3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63462156, jitterRate=-0.0543401837348938}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:17:18,189 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:18,190 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 9376883f3ccddf1028e50b0de6a8fad3: Running coprocessor pre-open hook at 1733203038178Writing region info on filesystem at 1733203038178Initializing all the Stores at 1733203038179 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203038179Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203038179Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203038179Cleaning up temporary data from old regions at 1733203038186 (+7 ms)Running coprocessor post-open hooks at 1733203038189 (+3 ms)Region opened successfully at 1733203038190 (+1 ms) 2024-12-03T05:17:18,191 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., pid=144, masterSystemTime=1733203038173 2024-12-03T05:17:18,192 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:18,192 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:18,193 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:18,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:17:18,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T05:17:18,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 in 174 msec 2024-12-03T05:17:18,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-03T05:17:18,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, ASSIGN in 330 msec 2024-12-03T05:17:18,206 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:17:18,207 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203038206"}]},"ts":"1733203038206"} 2024-12-03T05:17:18,208 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T05:17:18,215 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:17:18,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 773 msec 2024-12-03T05:17:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T05:17:18,580 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:18,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31094bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:18,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:18,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:18,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7ef3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:18,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3155064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:18,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:18,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:18,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:17:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,44191,1733202886389 2024-12-03T05:17:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3299bb0d 2024-12-03T05:17:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:17:18,591 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:17:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T05:17:18,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:17:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742395_1571 (size=1097) 2024-12-03T05:17:19,041 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T05:17:19,041 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T05:17:19,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:17:19,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE}] 2024-12-03T05:17:19,054 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE 2024-12-03T05:17:19,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE because future has completed 2024-12-03T05:17:19,057 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:17:19,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:17:19,210 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,210 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:17:19,210 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 9376883f3ccddf1028e50b0de6a8fad3, disabling compactions & flushes 2024-12-03T05:17:19,210 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,211 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,211 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. after waiting 0 ms 2024-12-03T05:17:19,211 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,214 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T05:17:19,215 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,215 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 9376883f3ccddf1028e50b0de6a8fad3: Waiting for close lock at 1733203039210Running coprocessor pre-close hooks at 1733203039210Disabling compacts and flushes for region at 1733203039210Disabling writes for close at 1733203039211 (+1 ms)Writing region close event to WAL at 1733203039211Running coprocessor post-close hooks at 1733203039215 (+4 ms)Closed at 1733203039215 2024-12-03T05:17:19,215 WARN [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 9376883f3ccddf1028e50b0de6a8fad3 to self. 2024-12-03T05:17:19,217 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=CLOSED 2024-12-03T05:17:19,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:17:19,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-03T05:17:19,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 in 163 msec 2024-12-03T05:17:19,223 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE; state=CLOSED, location=e3c6570475b6,37863,1733202887099; forceNewPlan=false, retain=true 2024-12-03T05:17:19,374 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=OPENING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE because future has completed 2024-12-03T05:17:19,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:17:19,531 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,531 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:17:19,531 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,532 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:17:19,532 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,532 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,533 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,534 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:19,534 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName A 2024-12-03T05:17:19,536 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:19,536 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:19,536 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,537 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:19,537 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName B 2024-12-03T05:17:19,537 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:19,538 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:19,538 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,538 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T05:17:19,538 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9376883f3ccddf1028e50b0de6a8fad3 columnFamilyName C 2024-12-03T05:17:19,538 DEBUG [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:19,539 INFO [StoreOpener-9376883f3ccddf1028e50b0de6a8fad3-1 {}] regionserver.HStore(327): Store=9376883f3ccddf1028e50b0de6a8fad3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:17:19,539 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,539 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,539 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,540 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,541 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,541 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,541 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:17:19,542 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,543 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 9376883f3ccddf1028e50b0de6a8fad3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63602063, jitterRate=-0.052255406975746155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:17:19,543 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,544 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 9376883f3ccddf1028e50b0de6a8fad3: Running coprocessor pre-open hook at 1733203039532Writing region info on filesystem at 1733203039532Initializing all the Stores at 1733203039533 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203039533Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203039533Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203039533Cleaning up temporary data from old regions at 1733203039541 (+8 ms)Running coprocessor post-open hooks at 1733203039543 (+2 ms)Region opened successfully at 1733203039544 (+1 ms) 2024-12-03T05:17:19,545 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., pid=149, masterSystemTime=1733203039528 2024-12-03T05:17:19,548 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,548 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,548 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=OPEN, openSeqNum=5, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:17:19,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-03T05:17:19,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 in 175 msec 2024-12-03T05:17:19,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T05:17:19,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, REOPEN/MOVE in 503 msec 2024-12-03T05:17:19,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T05:17:19,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-12-03T05:17:19,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 987 msec 2024-12-03T05:17:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T05:17:19,583 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:19,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea59a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@248232e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc0ac1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b93a72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe156e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6322a591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2948979e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,590 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bcab35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,591 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e3b374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e715d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42895800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266460f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b5dc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,597 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@956d4a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f9db7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d78280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69228a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c89c532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,611 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf38cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,613 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,615 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a9c7bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d60e004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7833988d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,631 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@881f8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c2683d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc7dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,44191,-1 for getting cluster id 2024-12-03T05:17:19,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:17:19,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a19201d6-9b21-461c-a6a9-e8f3e280f2f2' 2024-12-03T05:17:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:17:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a19201d6-9b21-461c-a6a9-e8f3e280f2f2" 2024-12-03T05:17:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c363bb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,44191,-1] 2024-12-03T05:17:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:17:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:19,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:17:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1476ea30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:17:19,638 DEBUG [Thread-2529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,639 DEBUG [Thread-2537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,640 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:19,641 DEBUG [Thread-2544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T05:17:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:19,647 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:19,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:19,648 DEBUG [Thread-2535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,650 DEBUG [Thread-2535 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,650 DEBUG [Thread-2535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,651 DEBUG [Thread-2533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,651 DEBUG [Thread-2540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,651 DEBUG [Thread-2548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,652 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,652 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,655 DEBUG [Thread-2546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:17:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,37863,1733202887099, seqNum=-1] 2024-12-03T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:17:19,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:17:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:19,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5] 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203039680, max timestamp=1733203039696Num uniques 859; 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203039680, max timestamp=1733203039696Num uniques 859; 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203039680, max timestamp=1733203039696Num uniques 859; 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:19,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:19,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f77ce449c1624cbbb2d82b57eac857c3_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203039658/Put/seqid=0 2024-12-03T05:17:19,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203099733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203099738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203099738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203099741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203099743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742396_1572 (size=12154) 2024-12-03T05:17:19,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203099850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203099860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203099860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203099860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203099861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:19,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:19,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203100060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203100070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203100071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203100071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203100071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:20,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f77ce449c1624cbbb2d82b57eac857c3_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f77ce449c1624cbbb2d82b57eac857c3_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:20,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/89253c8258434ec388dbb3d9162beaf0, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:20,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/89253c8258434ec388dbb3d9162beaf0 is 175, key is test_row_0/A:col10/1733203039658/Put/seqid=0 2024-12-03T05:17:20,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742397_1573 (size=30955) 2024-12-03T05:17:20,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/89253c8258434ec388dbb3d9162beaf0 2024-12-03T05:17:20,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f61d9af774004e53a4c8037a8f34b9b8 is 50, key is test_row_0/B:col10/1733203039658/Put/seqid=0 2024-12-03T05:17:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742398_1574 (size=12001) 2024-12-03T05:17:20,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f61d9af774004e53a4c8037a8f34b9b8 2024-12-03T05:17:20,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/72ea4663107f43229e6ff845f29a00d9 is 50, key is test_row_0/C:col10/1733203039658/Put/seqid=0 2024-12-03T05:17:20,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742399_1575 (size=12001) 2024-12-03T05:17:20,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/72ea4663107f43229e6ff845f29a00d9 2024-12-03T05:17:20,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/89253c8258434ec388dbb3d9162beaf0 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0 2024-12-03T05:17:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:20,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0, entries=150, sequenceid=15, filesize=30.2 K 2024-12-03T05:17:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f61d9af774004e53a4c8037a8f34b9b8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8 2024-12-03T05:17:20,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T05:17:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/72ea4663107f43229e6ff845f29a00d9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9 2024-12-03T05:17:20,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T05:17:20,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9376883f3ccddf1028e50b0de6a8fad3 in 633ms, sequenceid=15, compaction requested=false 2024-12-03T05:17:20,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:20,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:20,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120362f1faffaa6543a696ccd5d345d1ea43_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203039739/Put/seqid=0 2024-12-03T05:17:20,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742400_1576 (size=14594) 2024-12-03T05:17:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203100385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203100387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203100391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203100391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203100392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,400 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:20,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120362f1faffaa6543a696ccd5d345d1ea43_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120362f1faffaa6543a696ccd5d345d1ea43_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:20,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ac1475db1c5b4c14816b49bf0858cc6c, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:20,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ac1475db1c5b4c14816b49bf0858cc6c is 175, key is test_row_0/A:col10/1733203039739/Put/seqid=0 2024-12-03T05:17:20,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742401_1577 (size=39549) 2024-12-03T05:17:20,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ac1475db1c5b4c14816b49bf0858cc6c 2024-12-03T05:17:20,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/238ef0b736064237b4ca36ccf1cc4cf9 is 50, key is test_row_0/B:col10/1733203039739/Put/seqid=0 2024-12-03T05:17:20,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742402_1578 (size=12001) 2024-12-03T05:17:20,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/238ef0b736064237b4ca36ccf1cc4cf9 2024-12-03T05:17:20,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/17a76be20add4d2d8bf62cc08beebc2f is 50, key is test_row_0/C:col10/1733203039739/Put/seqid=0 2024-12-03T05:17:20,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742403_1579 (size=12001) 2024-12-03T05:17:20,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203100500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203100500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203100500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203100500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203100711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203100711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203100711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203100711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:20,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/17a76be20add4d2d8bf62cc08beebc2f 2024-12-03T05:17:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ac1475db1c5b4c14816b49bf0858cc6c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c 2024-12-03T05:17:20,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c, entries=200, sequenceid=52, filesize=38.6 K 2024-12-03T05:17:20,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/238ef0b736064237b4ca36ccf1cc4cf9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9 2024-12-03T05:17:20,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:20,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:20,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9, entries=150, sequenceid=52, filesize=11.7 K 2024-12-03T05:17:20,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/17a76be20add4d2d8bf62cc08beebc2f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f 2024-12-03T05:17:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203100918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:20,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f, entries=150, sequenceid=52, filesize=11.7 K 2024-12-03T05:17:20,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 9376883f3ccddf1028e50b0de6a8fad3 in 555ms, sequenceid=52, compaction requested=false 2024-12-03T05:17:20,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:21,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:21,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:21,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bfae11a9bec049f09c0f05b9b754d8ed_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:21,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742404_1580 (size=12154) 2024-12-03T05:17:21,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:21,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203101045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203101047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203101050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203101051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bfae11a9bec049f09c0f05b9b754d8ed_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bfae11a9bec049f09c0f05b9b754d8ed_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:21,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/a00aa6cf93b34c75b275487813d57e9d, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:21,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/a00aa6cf93b34c75b275487813d57e9d is 175, key is test_row_0/A:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:21,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742405_1581 (size=30955) 2024-12-03T05:17:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203101161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203101161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203101161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203101161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203101370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203101371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203101371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203101371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/a00aa6cf93b34c75b275487813d57e9d 2024-12-03T05:17:21,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5b8d3142f0254bbfba5dfdd0f968653e is 50, key is test_row_0/B:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:21,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742406_1582 (size=12001) 2024-12-03T05:17:21,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203101680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203101680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203101680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203101681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:21,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203101930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:21,933 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_0' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:21,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5b8d3142f0254bbfba5dfdd0f968653e 2024-12-03T05:17:21,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/0b5b8844e9854cb7aece6418e606ea0d is 50, key is test_row_0/C:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:21,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:17:21,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:21,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742407_1583 (size=12001) 2024-12-03T05:17:22,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/0b5b8844e9854cb7aece6418e606ea0d 2024-12-03T05:17:22,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/a00aa6cf93b34c75b275487813d57e9d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d 2024-12-03T05:17:22,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d, entries=150, sequenceid=65, filesize=30.2 K 2024-12-03T05:17:22,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5b8d3142f0254bbfba5dfdd0f968653e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e 2024-12-03T05:17:22,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T05:17:22,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/0b5b8844e9854cb7aece6418e606ea0d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d 2024-12-03T05:17:22,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T05:17:22,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9376883f3ccddf1028e50b0de6a8fad3 in 1034ms, sequenceid=65, compaction requested=true 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:22,058 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:22,058 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:22,061 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:22,061 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:22,061 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=99.1 K 2024-12-03T05:17:22,061 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=35.2 K 2024-12-03T05:17:22,061 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d] 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89253c8258434ec388dbb3d9162beaf0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733203039645 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f61d9af774004e53a4c8037a8f34b9b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733203039645 2024-12-03T05:17:22,061 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac1475db1c5b4c14816b49bf0858cc6c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733203039731 2024-12-03T05:17:22,062 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 238ef0b736064237b4ca36ccf1cc4cf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733203039731 2024-12-03T05:17:22,062 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00aa6cf93b34c75b275487813d57e9d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:22,062 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8d3142f0254bbfba5dfdd0f968653e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:22,071 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:22,080 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#495 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:22,081 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/897f33e6b53841be83ba41ef3027a6e7 is 50, key is test_row_0/B:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:22,088 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037e7a56e284dc451395d856f74ee493a7_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:22,090 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037e7a56e284dc451395d856f74ee493a7_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:22,090 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037e7a56e284dc451395d856f74ee493a7_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742409_1585 (size=4469) 2024-12-03T05:17:22,108 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#494 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:22,108 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/67bf51ba82ea4958bda5ae4560730e84 is 175, key is test_row_0/A:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:22,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742408_1584 (size=12104) 2024-12-03T05:17:22,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742410_1586 (size=31058) 2024-12-03T05:17:22,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T05:17:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:22,149 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036c0564d582034439ba8dd0cf00390a8c_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203041049/Put/seqid=0 2024-12-03T05:17:22,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742411_1587 (size=12154) 2024-12-03T05:17:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:22,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:22,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036c0564d582034439ba8dd0cf00390a8c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c0564d582034439ba8dd0cf00390a8c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/3340490f165a4b6a8781e5a6e5711bac, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/3340490f165a4b6a8781e5a6e5711bac is 175, key is test_row_0/A:col10/1733203041049/Put/seqid=0 2024-12-03T05:17:22,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203102212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203102212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203102217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203102220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742412_1588 (size=30955) 2024-12-03T05:17:22,232 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/3340490f165a4b6a8781e5a6e5711bac 2024-12-03T05:17:22,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a66ca028edb846fcac79dce2e42f5e9d is 50, key is test_row_0/B:col10/1733203041049/Put/seqid=0 2024-12-03T05:17:22,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742413_1589 (size=12001) 2024-12-03T05:17:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203102322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203102331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203102331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203102331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,518 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/897f33e6b53841be83ba41ef3027a6e7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/897f33e6b53841be83ba41ef3027a6e7 2024-12-03T05:17:22,520 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into 897f33e6b53841be83ba41ef3027a6e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:22,520 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:22,520 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=13, startTime=1733203042058; duration=0sec 2024-12-03T05:17:22,521 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:22,521 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:22,521 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:22,521 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:22,521 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:22,521 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:22,521 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=35.2 K 2024-12-03T05:17:22,522 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ea4663107f43229e6ff845f29a00d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733203039645 2024-12-03T05:17:22,522 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a76be20add4d2d8bf62cc08beebc2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733203039731 2024-12-03T05:17:22,522 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5b8844e9854cb7aece6418e606ea0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:22,529 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:22,529 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4218a28404fc44bbba9816ec32525894 is 50, key is test_row_0/C:col10/1733203041022/Put/seqid=0 2024-12-03T05:17:22,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203102530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742414_1590 (size=12104) 2024-12-03T05:17:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,536 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/67bf51ba82ea4958bda5ae4560730e84 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84 2024-12-03T05:17:22,539 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into 67bf51ba82ea4958bda5ae4560730e84(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:22,539 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:22,539 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=13, startTime=1733203042058; duration=0sec 2024-12-03T05:17:22,539 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:22,539 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,702 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a66ca028edb846fcac79dce2e42f5e9d 2024-12-03T05:17:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4dc7e6d50437447290da0c195f6805d3 is 50, key is test_row_0/C:col10/1733203041049/Put/seqid=0 2024-12-03T05:17:22,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742415_1591 (size=12001) 2024-12-03T05:17:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203102840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:22,955 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4218a28404fc44bbba9816ec32525894 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4218a28404fc44bbba9816ec32525894 2024-12-03T05:17:22,967 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into 4218a28404fc44bbba9816ec32525894(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:22,967 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:22,968 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=13, startTime=1733203042058; duration=0sec 2024-12-03T05:17:22,968 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:22,968 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:23,121 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4dc7e6d50437447290da0c195f6805d3 2024-12-03T05:17:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/3340490f165a4b6a8781e5a6e5711bac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac 2024-12-03T05:17:23,138 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac, entries=150, sequenceid=85, filesize=30.2 K 2024-12-03T05:17:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a66ca028edb846fcac79dce2e42f5e9d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d 2024-12-03T05:17:23,160 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d, entries=150, sequenceid=85, filesize=11.7 K 2024-12-03T05:17:23,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4dc7e6d50437447290da0c195f6805d3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3 2024-12-03T05:17:23,178 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3, entries=150, sequenceid=85, filesize=11.7 K 2024-12-03T05:17:23,180 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9376883f3ccddf1028e50b0de6a8fad3 in 1031ms, sequenceid=85, compaction requested=false 2024-12-03T05:17:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T05:17:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T05:17:23,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T05:17:23,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5370 sec 2024-12-03T05:17:23,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.5470 sec 2024-12-03T05:17:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038672ff3140a44651a2ea4de1f25e442c_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:23,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742416_1592 (size=14594) 2024-12-03T05:17:23,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:23,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203103380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038672ff3140a44651a2ea4de1f25e442c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038672ff3140a44651a2ea4de1f25e442c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203103388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203103389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,396 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bb4a675c11904a2bb843e233561bbfec, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:23,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bb4a675c11904a2bb843e233561bbfec is 175, key is test_row_0/A:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203103392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742417_1593 (size=39549) 2024-12-03T05:17:23,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bb4a675c11904a2bb843e233561bbfec 2024-12-03T05:17:23,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/969f1e4579d94ce28e1bd6aceef1e9a3 is 50, key is test_row_0/B:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:23,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742418_1594 (size=12001) 2024-12-03T05:17:23,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/969f1e4579d94ce28e1bd6aceef1e9a3 2024-12-03T05:17:23,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/1fa6bc97faae45239113f13db53dc9aa is 50, key is test_row_0/C:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742419_1595 (size=12001) 2024-12-03T05:17:23,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203103501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203103501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203103501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203103510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203103710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203103710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203103711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203103721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T05:17:23,811 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:23,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T05:17:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:23,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:23,815 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:23,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:23,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/1fa6bc97faae45239113f13db53dc9aa 2024-12-03T05:17:23,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bb4a675c11904a2bb843e233561bbfec as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec 2024-12-03T05:17:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:23,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec, entries=200, sequenceid=101, filesize=38.6 K 2024-12-03T05:17:23,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/969f1e4579d94ce28e1bd6aceef1e9a3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3 2024-12-03T05:17:23,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T05:17:23,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/1fa6bc97faae45239113f13db53dc9aa as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa 2024-12-03T05:17:23,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T05:17:23,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9376883f3ccddf1028e50b0de6a8fad3 in 587ms, sequenceid=101, compaction requested=true 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:23,940 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:23,940 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:23,941 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,941 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,941 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/897f33e6b53841be83ba41ef3027a6e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=35.3 K 2024-12-03T05:17:23,941 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=99.2 K 2024-12-03T05:17:23,941 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec] 2024-12-03T05:17:23,941 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 897f33e6b53841be83ba41ef3027a6e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:23,942 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a66ca028edb846fcac79dce2e42f5e9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733203041033 2024-12-03T05:17:23,942 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67bf51ba82ea4958bda5ae4560730e84, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:23,942 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 969f1e4579d94ce28e1bd6aceef1e9a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:23,942 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3340490f165a4b6a8781e5a6e5711bac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733203041033 2024-12-03T05:17:23,942 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb4a675c11904a2bb843e233561bbfec, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:23,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:23,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:23,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:23,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eca8599d05744ef6ad201d3cb1e30d1a_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203043951/Put/seqid=0 2024-12-03T05:17:23,980 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:23,981 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6f25e37634d2489eaded68f08d359d84 is 50, key is test_row_0/B:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:23,982 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:23,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742420_1596 (size=12207) 2024-12-03T05:17:23,985 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120345ee7ab0d4d74074bed1ea8ff9ebb9ac_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:23,987 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120345ee7ab0d4d74074bed1ea8ff9ebb9ac_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:23,987 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120345ee7ab0d4d74074bed1ea8ff9ebb9ac_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:23,992 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6f25e37634d2489eaded68f08d359d84 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6f25e37634d2489eaded68f08d359d84 2024-12-03T05:17:24,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203103999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,009 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into 6f25e37634d2489eaded68f08d359d84(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:24,009 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:24,010 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=13, startTime=1733203043940; duration=0sec 2024-12-03T05:17:24,010 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:24,010 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:24,010 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:24,011 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:24,011 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:24,011 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,012 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4218a28404fc44bbba9816ec32525894, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=35.3 K 2024-12-03T05:17:24,012 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4218a28404fc44bbba9816ec32525894, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733203040388 2024-12-03T05:17:24,013 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc7e6d50437447290da0c195f6805d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733203041033 2024-12-03T05:17:24,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742421_1597 (size=14594) 2024-12-03T05:17:24,014 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa6bc97faae45239113f13db53dc9aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:24,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:24,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eca8599d05744ef6ad201d3cb1e30d1a_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eca8599d05744ef6ad201d3cb1e30d1a_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742422_1598 (size=4469) 2024-12-03T05:17:24,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/80b444afdeac4bf78e392e08fad838c6, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:24,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/80b444afdeac4bf78e392e08fad838c6 is 175, key is test_row_0/A:col10/1733203043951/Put/seqid=0 2024-12-03T05:17:24,020 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#505 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:24,020 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1d1898227bd54ed4b4394740aaabc570 is 175, key is test_row_0/A:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:24,021 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:24,022 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/db86d31a19e4436f916a8d135e7739a4 is 50, key is test_row_0/C:col10/1733203043352/Put/seqid=0 2024-12-03T05:17:24,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742425_1601 (size=12207) 2024-12-03T05:17:24,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742423_1599 (size=39549) 2024-12-03T05:17:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742424_1600 (size=31161) 2024-12-03T05:17:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203104021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203104021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203104022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,029 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/db86d31a19e4436f916a8d135e7739a4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/db86d31a19e4436f916a8d135e7739a4 2024-12-03T05:17:24,029 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1d1898227bd54ed4b4394740aaabc570 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570 2024-12-03T05:17:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203104031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,036 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into db86d31a19e4436f916a8d135e7739a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:24,036 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:24,036 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=13, startTime=1733203043940; duration=0sec 2024-12-03T05:17:24,036 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:24,036 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:24,039 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into 1d1898227bd54ed4b4394740aaabc570(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:24,039 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:24,039 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=13, startTime=1733203043940; duration=0sec 2024-12-03T05:17:24,039 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:24,039 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203104110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:24,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203104320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/80b444afdeac4bf78e392e08fad838c6 2024-12-03T05:17:24,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/81604493ed954ce5935d272207a7a49a is 50, key is test_row_0/B:col10/1733203043951/Put/seqid=0 2024-12-03T05:17:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742426_1602 (size=12001) 2024-12-03T05:17:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203104530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203104530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203104530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203104541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203104630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:24,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/81604493ed954ce5935d272207a7a49a 2024-12-03T05:17:24,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/92fa0b76fa7f438c91f17fb2d5bf7852 is 50, key is test_row_0/C:col10/1733203043951/Put/seqid=0 2024-12-03T05:17:24,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:24,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:24,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742427_1603 (size=12001) 2024-12-03T05:17:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:25,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:25,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203105143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:25,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:25,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/92fa0b76fa7f438c91f17fb2d5bf7852 2024-12-03T05:17:25,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/80b444afdeac4bf78e392e08fad838c6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6 2024-12-03T05:17:25,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6, entries=200, sequenceid=121, filesize=38.6 K 2024-12-03T05:17:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/81604493ed954ce5935d272207a7a49a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a 2024-12-03T05:17:25,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T05:17:25,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/92fa0b76fa7f438c91f17fb2d5bf7852 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852 2024-12-03T05:17:25,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T05:17:25,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9376883f3ccddf1028e50b0de6a8fad3 in 1395ms, sequenceid=121, compaction requested=false 2024-12-03T05:17:25,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:25,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:25,395 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035593d48d1ed74ff1847477a014910f91_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203043971/Put/seqid=0 2024-12-03T05:17:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742428_1604 (size=12204) 2024-12-03T05:17:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:25,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:25,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203105589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203105589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203105591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203105592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203105701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203105701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203105701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203105701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:25,829 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035593d48d1ed74ff1847477a014910f91_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035593d48d1ed74ff1847477a014910f91_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:25,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ed4a1f58910c430695cbf788f3906799, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:25,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ed4a1f58910c430695cbf788f3906799 is 175, key is test_row_0/A:col10/1733203043971/Put/seqid=0 2024-12-03T05:17:25,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742429_1605 (size=31005) 2024-12-03T05:17:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ed4a1f58910c430695cbf788f3906799 2024-12-03T05:17:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/36666b4b6dec492c9cd22d2e506eaaab is 50, key is test_row_0/B:col10/1733203043971/Put/seqid=0 2024-12-03T05:17:25,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742430_1606 (size=12051) 2024-12-03T05:17:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203105910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203105910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203105910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203105910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203106164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,168 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203106220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203106220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203106220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203106221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,241 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/36666b4b6dec492c9cd22d2e506eaaab 2024-12-03T05:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/342ffb3f967f47baad66ba857749f444 is 50, key is test_row_0/C:col10/1733203043971/Put/seqid=0 2024-12-03T05:17:26,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742431_1607 (size=12051) 2024-12-03T05:17:26,653 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/342ffb3f967f47baad66ba857749f444 2024-12-03T05:17:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/ed4a1f58910c430695cbf788f3906799 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799 2024-12-03T05:17:26,685 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799, entries=150, sequenceid=136, filesize=30.3 K 2024-12-03T05:17:26,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/36666b4b6dec492c9cd22d2e506eaaab as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab 2024-12-03T05:17:26,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab, entries=150, sequenceid=136, filesize=11.8 K 2024-12-03T05:17:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/342ffb3f967f47baad66ba857749f444 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444 2024-12-03T05:17:26,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T05:17:26,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T05:17:26,719 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444, entries=150, sequenceid=136, filesize=11.8 K 2024-12-03T05:17:26,720 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9376883f3ccddf1028e50b0de6a8fad3 in 1325ms, sequenceid=136, compaction requested=true 2024-12-03T05:17:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T05:17:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T05:17:26,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T05:17:26,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9090 sec 2024-12-03T05:17:26,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.9190 sec 2024-12-03T05:17:26,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T05:17:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:26,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa409b30b2d846a6b1d88f4c4c94da3c_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:26,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203106766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203106768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203106772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203106775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742432_1608 (size=17284) 2024-12-03T05:17:26,805 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:26,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa409b30b2d846a6b1d88f4c4c94da3c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa409b30b2d846a6b1d88f4c4c94da3c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:26,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/5f0b6b7d76f74060824a58d19d7351a2, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:26,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/5f0b6b7d76f74060824a58d19d7351a2 is 175, key is test_row_0/A:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:26,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742433_1609 (size=48389) 2024-12-03T05:17:26,855 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/5f0b6b7d76f74060824a58d19d7351a2 2024-12-03T05:17:26,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/0eead8776108475f91004e01c227ef3c is 50, key is test_row_0/B:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203106881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203106881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203106890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203106892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:26,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742434_1610 (size=12151) 2024-12-03T05:17:26,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/0eead8776108475f91004e01c227ef3c 2024-12-03T05:17:26,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/97444dccebcb4d9997646fb5230ccf28 is 50, key is test_row_0/C:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:26,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742435_1611 (size=12151) 2024-12-03T05:17:26,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/97444dccebcb4d9997646fb5230ccf28 2024-12-03T05:17:26,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/5f0b6b7d76f74060824a58d19d7351a2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2 2024-12-03T05:17:26,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2, entries=250, sequenceid=158, filesize=47.3 K 2024-12-03T05:17:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/0eead8776108475f91004e01c227ef3c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c 2024-12-03T05:17:26,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c, entries=150, sequenceid=158, filesize=11.9 K 2024-12-03T05:17:26,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/97444dccebcb4d9997646fb5230ccf28 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28 2024-12-03T05:17:26,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28, entries=150, sequenceid=158, filesize=11.9 K 2024-12-03T05:17:26,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 9376883f3ccddf1028e50b0de6a8fad3 in 262ms, sequenceid=158, compaction requested=true 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:26,997 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:26,997 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:26,999 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:17:26,999 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:17:26,999 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:26,999 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:26,999 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:26,999 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:26,999 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=146.6 K 2024-12-03T05:17:26,999 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6f25e37634d2489eaded68f08d359d84, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=47.3 K 2024-12-03T05:17:26,999 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:26,999 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2] 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1898227bd54ed4b4394740aaabc570, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f25e37634d2489eaded68f08d359d84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 81604493ed954ce5935d272207a7a49a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733203043375 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b444afdeac4bf78e392e08fad838c6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733203043375 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 36666b4b6dec492c9cd22d2e506eaaab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733203043971 2024-12-03T05:17:27,000 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4a1f58910c430695cbf788f3906799, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733203043971 2024-12-03T05:17:27,001 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eead8776108475f91004e01c227ef3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:27,003 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0b6b7d76f74060824a58d19d7351a2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:27,010 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:27,012 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b7e2e216ac524c298ba28d8f66c30923_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:27,013 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:27,014 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/23cf691aaccb4a08ab1c91bb6880542b is 50, key is test_row_0/B:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:27,014 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b7e2e216ac524c298ba28d8f66c30923_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:27,014 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b7e2e216ac524c298ba28d8f66c30923_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:27,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742437_1613 (size=4469) 2024-12-03T05:17:27,037 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#515 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:27,037 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/43f64ea0f3b146ce8d25b13f466a3dc7 is 175, key is test_row_0/A:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:27,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742436_1612 (size=12493) 2024-12-03T05:17:27,051 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/23cf691aaccb4a08ab1c91bb6880542b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/23cf691aaccb4a08ab1c91bb6880542b 2024-12-03T05:17:27,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742438_1614 (size=31447) 2024-12-03T05:17:27,060 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into 23cf691aaccb4a08ab1c91bb6880542b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:27,060 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:27,060 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=12, startTime=1733203046997; duration=0sec 2024-12-03T05:17:27,060 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:27,060 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:27,061 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:17:27,064 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T05:17:27,064 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:27,064 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:27,065 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/db86d31a19e4436f916a8d135e7739a4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=47.3 K 2024-12-03T05:17:27,065 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting db86d31a19e4436f916a8d135e7739a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733203042208 2024-12-03T05:17:27,065 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 92fa0b76fa7f438c91f17fb2d5bf7852, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733203043375 2024-12-03T05:17:27,065 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 342ffb3f967f47baad66ba857749f444, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733203043971 2024-12-03T05:17:27,066 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 97444dccebcb4d9997646fb5230ccf28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:27,072 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/43f64ea0f3b146ce8d25b13f466a3dc7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7 2024-12-03T05:17:27,078 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into 43f64ea0f3b146ce8d25b13f466a3dc7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:27,078 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:27,078 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=12, startTime=1733203046997; duration=0sec 2024-12-03T05:17:27,078 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:27,078 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:27,088 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:27,088 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/6b77765680024feeb070d1f76857fa48 is 50, key is test_row_0/C:col10/1733203045589/Put/seqid=0 2024-12-03T05:17:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:27,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:27,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203694dd082edf54923b93821130598d760_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203046757/Put/seqid=0 2024-12-03T05:17:27,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742440_1616 (size=12304) 2024-12-03T05:17:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742439_1615 (size=12493) 2024-12-03T05:17:27,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:27,149 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/6b77765680024feeb070d1f76857fa48 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6b77765680024feeb070d1f76857fa48 2024-12-03T05:17:27,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203694dd082edf54923b93821130598d760_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203694dd082edf54923b93821130598d760_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203107145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203107144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203047120, max timestamp=1733203047149Num uniques 859; 2024-12-03T05:17:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bd7605bde1a548b485821daa8beafbc6, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:27,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bd7605bde1a548b485821daa8beafbc6 is 175, key is test_row_0/A:col10/1733203046757/Put/seqid=0 2024-12-03T05:17:27,158 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into 6b77765680024feeb070d1f76857fa48(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:27,158 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:27,158 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=12, startTime=1733203046997; duration=0sec 2024-12-03T05:17:27,158 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:27,158 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:27,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742441_1617 (size=31105) 2024-12-03T05:17:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203107165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203107169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203107261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203107261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203107280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203107280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203107470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203107471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203107491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203107491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,563 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bd7605bde1a548b485821daa8beafbc6 2024-12-03T05:17:27,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/ca5d0df8ef664221864188add0019345 is 50, key is test_row_0/B:col10/1733203046757/Put/seqid=0 2024-12-03T05:17:27,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742442_1618 (size=12151) 2024-12-03T05:17:27,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203107781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203107781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203107801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203107804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T05:17:27,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:27,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T05:17:27,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:27,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:27,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:28,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/ca5d0df8ef664221864188add0019345 2024-12-03T05:17:28,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a9afa71a994491f86ec5e00f38d9ea7 is 50, key is test_row_0/C:col10/1733203046757/Put/seqid=0 2024-12-03T05:17:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:28,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742443_1619 (size=12151) 2024-12-03T05:17:28,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a9afa71a994491f86ec5e00f38d9ea7 2024-12-03T05:17:28,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T05:17:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:28,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:28,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/bd7605bde1a548b485821daa8beafbc6 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6 2024-12-03T05:17:28,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6, entries=150, sequenceid=171, filesize=30.4 K 2024-12-03T05:17:28,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/ca5d0df8ef664221864188add0019345 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345 2024-12-03T05:17:28,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203108181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,188 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T05:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a9afa71a994491f86ec5e00f38d9ea7 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7 2024-12-03T05:17:28,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7, entries=150, sequenceid=171, filesize=11.9 K 2024-12-03T05:17:28,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 9376883f3ccddf1028e50b0de6a8fad3 in 1113ms, sequenceid=171, compaction requested=false 2024-12-03T05:17:28,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:28,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T05:17:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:28,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=125.23 KB heapSize=366.80 KB 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:28,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f61b5cab0f94387ab49800e049931f6_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203047162/Put/seqid=0 2024-12-03T05:17:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742444_1620 (size=12304) 2024-12-03T05:17:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203108311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203108311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203108315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203108320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203108421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203108421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203108431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:28,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203108630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203108640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203108640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:28,698 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f61b5cab0f94387ab49800e049931f6_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f61b5cab0f94387ab49800e049931f6_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:28,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b63332aab9dc45adb2fb17c3329d295f, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b63332aab9dc45adb2fb17c3329d295f is 175, key is test_row_0/A:col10/1733203047162/Put/seqid=0 2024-12-03T05:17:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742445_1621 (size=31105) 2024-12-03T05:17:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203108940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203108951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203108951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:29,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b63332aab9dc45adb2fb17c3329d295f 2024-12-03T05:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f62e4a6bfe15461b8d747994ee105805 is 50, key is test_row_0/B:col10/1733203047162/Put/seqid=0 2024-12-03T05:17:29,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742446_1622 (size=12151) 2024-12-03T05:17:29,189 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f62e4a6bfe15461b8d747994ee105805 2024-12-03T05:17:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/33ad410f257544278f565af0123b7916 is 50, key is test_row_0/C:col10/1733203047162/Put/seqid=0 2024-12-03T05:17:29,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742447_1623 (size=12151) 2024-12-03T05:17:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203109341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:29,344 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:29,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203109450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:29,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:29,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203109461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:29,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203109461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:29,626 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/33ad410f257544278f565af0123b7916 2024-12-03T05:17:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b63332aab9dc45adb2fb17c3329d295f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f 2024-12-03T05:17:29,633 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f, entries=150, sequenceid=198, filesize=30.4 K 2024-12-03T05:17:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/f62e4a6bfe15461b8d747994ee105805 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805 2024-12-03T05:17:29,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T05:17:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/33ad410f257544278f565af0123b7916 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916 2024-12-03T05:17:29,641 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T05:17:29,642 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~366.75 KB/375552, currentSize=53.67 KB/54960 for 9376883f3ccddf1028e50b0de6a8fad3 in 1358ms, sequenceid=198, compaction requested=true 2024-12-03T05:17:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T05:17:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T05:17:29,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T05:17:29,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2024-12-03T05:17:29,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.6750 sec 2024-12-03T05:17:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T05:17:30,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:30,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-03T05:17:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:30,123 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:30,123 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:30,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:30,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T05:17:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:30,276 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:30,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037f459f9db27c461fb7119e156ac0d077_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203048309/Put/seqid=0 2024-12-03T05:17:30,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742448_1624 (size=12304) 2024-12-03T05:17:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:30,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203050499, max timestamp=1733203050514Num uniques 859; 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203050499, max timestamp=1733203050514Num uniques 859; 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733203050499, max timestamp=1733203050514Num uniques 859; 2024-12-03T05:17:30,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T05:17:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203110540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203110541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203110541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203110651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203110652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203110652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:30,704 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037f459f9db27c461fb7119e156ac0d077_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f459f9db27c461fb7119e156ac0d077_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:30,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/c3811ea6098147ab84ca518d58011c2e, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/c3811ea6098147ab84ca518d58011c2e is 175, key is test_row_0/A:col10/1733203048309/Put/seqid=0 2024-12-03T05:17:30,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742449_1625 (size=31105) 2024-12-03T05:17:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203110865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203110865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:30,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203110870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,122 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/c3811ea6098147ab84ca518d58011c2e 2024-12-03T05:17:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6870471218924a1b8a342a2fb7a72d1a is 50, key is test_row_0/B:col10/1733203048309/Put/seqid=0 2024-12-03T05:17:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203111171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203111180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203111180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742450_1626 (size=12151) 2024-12-03T05:17:31,211 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6870471218924a1b8a342a2fb7a72d1a 2024-12-03T05:17:31,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/3f30ffaf4d924d3baad8ea4cbff8f5ac is 50, key is test_row_0/C:col10/1733203048309/Put/seqid=0 2024-12-03T05:17:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:31,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742451_1627 (size=12151) 2024-12-03T05:17:31,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/3f30ffaf4d924d3baad8ea4cbff8f5ac 2024-12-03T05:17:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/c3811ea6098147ab84ca518d58011c2e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e 2024-12-03T05:17:31,370 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e, entries=150, sequenceid=209, filesize=30.4 K 2024-12-03T05:17:31,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6870471218924a1b8a342a2fb7a72d1a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a 2024-12-03T05:17:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203111370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,374 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,384 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T05:17:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/3f30ffaf4d924d3baad8ea4cbff8f5ac as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac 2024-12-03T05:17:31,398 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T05:17:31,404 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9376883f3ccddf1028e50b0de6a8fad3 in 1128ms, sequenceid=209, compaction requested=true 2024-12-03T05:17:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T05:17:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T05:17:31,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T05:17:31,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-12-03T05:17:31,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.2870 sec 2024-12-03T05:17:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:31,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:31,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033fe273ccc564499cbd89ae45579c93f1_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:31,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742452_1628 (size=14794) 2024-12-03T05:17:31,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203111708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203111710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203111710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203111821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203111821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203111821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203112040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203112040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203112040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:32,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033fe273ccc564499cbd89ae45579c93f1_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033fe273ccc564499cbd89ae45579c93f1_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:32,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/45e8825a986e4e00a62995bcdbd19eba, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:32,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/45e8825a986e4e00a62995bcdbd19eba is 175, key is test_row_0/A:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742453_1629 (size=39749) 2024-12-03T05:17:32,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/45e8825a986e4e00a62995bcdbd19eba 2024-12-03T05:17:32,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/03dab5429d164bdfb556b5368eb8fcab is 50, key is test_row_0/B:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742454_1630 (size=12151) 2024-12-03T05:17:32,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/03dab5429d164bdfb556b5368eb8fcab 2024-12-03T05:17:32,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/bfed0f86ed4e47229b0dc48020f6efad is 50, key is test_row_0/C:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59254 deadline: 1733203112231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,232 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_1' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742455_1631 (size=12151) 2024-12-03T05:17:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T05:17:32,271 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:32,272 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-03T05:17:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:32,273 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:32,274 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:32,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:32,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203112350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203112350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203112350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:32,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T05:17:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T05:17:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:32,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/bfed0f86ed4e47229b0dc48020f6efad 2024-12-03T05:17:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/45e8825a986e4e00a62995bcdbd19eba as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba 2024-12-03T05:17:32,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba, entries=200, sequenceid=245, filesize=38.8 K 2024-12-03T05:17:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/03dab5429d164bdfb556b5368eb8fcab as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab 2024-12-03T05:17:32,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab, entries=150, sequenceid=245, filesize=11.9 K 2024-12-03T05:17:32,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/bfed0f86ed4e47229b0dc48020f6efad as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad 2024-12-03T05:17:32,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad, entries=150, sequenceid=245, filesize=11.9 K 2024-12-03T05:17:32,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9376883f3ccddf1028e50b0de6a8fad3 in 985ms, sequenceid=245, compaction requested=true 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:32,669 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:32,669 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61097 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:32,671 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,671 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,671 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/23cf691aaccb4a08ab1c91bb6880542b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=59.7 K 2024-12-03T05:17:32,671 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=160.7 K 2024-12-03T05:17:32,671 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba] 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 23cf691aaccb4a08ab1c91bb6880542b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:32,671 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43f64ea0f3b146ce8d25b13f466a3dc7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5d0df8ef664221864188add0019345, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733203046757 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7605bde1a548b485821daa8beafbc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733203046757 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting f62e4a6bfe15461b8d747994ee105805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733203047162 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63332aab9dc45adb2fb17c3329d295f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733203047162 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6870471218924a1b8a342a2fb7a72d1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733203048309 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3811ea6098147ab84ca518d58011c2e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733203048309 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 03dab5429d164bdfb556b5368eb8fcab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:32,672 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e8825a986e4e00a62995bcdbd19eba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:32,680 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:32,682 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:32,683 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6a123bef03c84b8b9dbe5cd3b184e8fd is 50, key is test_row_0/B:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,693 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f416c7a50e8644daa2c51a0da12b1364_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:32,696 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f416c7a50e8644daa2c51a0da12b1364_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:32,696 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f416c7a50e8644daa2c51a0da12b1364_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:32,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742457_1633 (size=4469) 2024-12-03T05:17:32,703 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#531 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:32,703 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/0617e0ef6c1a44968a1df7c74d8eeb57 is 175, key is test_row_0/A:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742456_1632 (size=12663) 2024-12-03T05:17:32,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,731 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2358af460d24511982ff591bad8158e_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203051702/Put/seqid=0 2024-12-03T05:17:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742458_1634 (size=31617) 2024-12-03T05:17:32,743 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/0617e0ef6c1a44968a1df7c74d8eeb57 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57 2024-12-03T05:17:32,749 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into 0617e0ef6c1a44968a1df7c74d8eeb57(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:32,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:32,749 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=11, startTime=1733203052669; duration=0sec 2024-12-03T05:17:32,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:32,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:32,749 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T05:17:32,752 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61097 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T05:17:32,752 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:32,752 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:32,752 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6b77765680024feeb070d1f76857fa48, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=59.7 K 2024-12-03T05:17:32,753 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b77765680024feeb070d1f76857fa48, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733203045573 2024-12-03T05:17:32,753 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a9afa71a994491f86ec5e00f38d9ea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733203046757 2024-12-03T05:17:32,753 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ad410f257544278f565af0123b7916, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733203047162 2024-12-03T05:17:32,753 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f30ffaf4d924d3baad8ea4cbff8f5ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733203048309 2024-12-03T05:17:32,753 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfed0f86ed4e47229b0dc48020f6efad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:32,766 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:32,766 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/17ae5ab1a80740e9951a99a143ef5ed3 is 50, key is test_row_0/C:col10/1733203050540/Put/seqid=0 2024-12-03T05:17:32,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742459_1635 (size=12304) 2024-12-03T05:17:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742460_1636 (size=12663) 2024-12-03T05:17:32,781 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/17ae5ab1a80740e9951a99a143ef5ed3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17ae5ab1a80740e9951a99a143ef5ed3 2024-12-03T05:17:32,785 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into 17ae5ab1a80740e9951a99a143ef5ed3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:32,785 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:32,785 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=11, startTime=1733203052669; duration=0sec 2024-12-03T05:17:32,785 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:32,786 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203052891, max timestamp=1733203052897Num uniques 859; 2024-12-03T05:17:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733203052880, max timestamp=1733203052897Num uniques 859; 2024-12-03T05:17:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T05:17:32,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203052891, max timestamp=1733203052897Num uniques 859; 2024-12-03T05:17:32,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203112922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203112926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:32,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203112927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203113031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203113040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203113040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,111 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/6a123bef03c84b8b9dbe5cd3b184e8fd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6a123bef03c84b8b9dbe5cd3b184e8fd 2024-12-03T05:17:33,114 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into 6a123bef03c84b8b9dbe5cd3b184e8fd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:33,114 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:33,114 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=11, startTime=1733203052669; duration=0sec 2024-12-03T05:17:33,114 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:33,114 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:33,177 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c2358af460d24511982ff591bad8158e_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2358af460d24511982ff591bad8158e_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:33,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/dae92d6eb01e46e2aceb49bc1d89c547, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/dae92d6eb01e46e2aceb49bc1d89c547 is 175, key is test_row_0/A:col10/1733203051702/Put/seqid=0 2024-12-03T05:17:33,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742461_1637 (size=31105) 2024-12-03T05:17:33,181 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/dae92d6eb01e46e2aceb49bc1d89c547 2024-12-03T05:17:33,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/cb5cfd38d46943b0be9676a3e8799d8a is 50, key is test_row_0/B:col10/1733203051702/Put/seqid=0 2024-12-03T05:17:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742462_1638 (size=12151) 2024-12-03T05:17:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203113240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203113251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203113251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:33,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203113550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203113570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203113570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/cb5cfd38d46943b0be9676a3e8799d8a 2024-12-03T05:17:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5a572f40fae84ab784ad50ed37835205 is 50, key is test_row_0/C:col10/1733203051702/Put/seqid=0 2024-12-03T05:17:33,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742463_1639 (size=12151) 2024-12-03T05:17:33,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5a572f40fae84ab784ad50ed37835205 2024-12-03T05:17:33,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/dae92d6eb01e46e2aceb49bc1d89c547 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547 2024-12-03T05:17:33,646 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547, entries=150, sequenceid=255, filesize=30.4 K 2024-12-03T05:17:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/cb5cfd38d46943b0be9676a3e8799d8a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a 2024-12-03T05:17:33,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T05:17:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5a572f40fae84ab784ad50ed37835205 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205 2024-12-03T05:17:33,660 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T05:17:33,661 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 9376883f3ccddf1028e50b0de6a8fad3 in 930ms, sequenceid=255, compaction requested=false 2024-12-03T05:17:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-03T05:17:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-03T05:17:33,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T05:17:33,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-12-03T05:17:33,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.3930 sec 2024-12-03T05:17:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:34,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:34,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c641017930a24e9fba9d2543fe627279_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:34,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203114086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742464_1640 (size=14994) 2024-12-03T05:17:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203114092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203114092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203114201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203114201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203114201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203114410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203114410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203114410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T05:17:34,421 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:34,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-03T05:17:34,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:34,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:34,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:34,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:34,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c641017930a24e9fba9d2543fe627279_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c641017930a24e9fba9d2543fe627279_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:34,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/fdfa642afaea46e49a532a2381a1263e, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:34,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/fdfa642afaea46e49a532a2381a1263e is 175, key is test_row_0/A:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:34,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742465_1641 (size=39949) 2024-12-03T05:17:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:34,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203114720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203114721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203114721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:34,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:34,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:34,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:34,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/fdfa642afaea46e49a532a2381a1263e 2024-12-03T05:17:34,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5d98f91aba0f4666a83f41e30a590bc3 is 50, key is test_row_0/B:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:34,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742466_1642 (size=12301) 2024-12-03T05:17:34,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5d98f91aba0f4666a83f41e30a590bc3 2024-12-03T05:17:34,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a12dca1eeb94e62bc6ce7fc67328fb4 is 50, key is test_row_0/C:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:35,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742467_1643 (size=12301) 2024-12-03T05:17:35,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:35,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:35,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:35,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203115230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203115230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:35,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203115230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:35,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:35,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a12dca1eeb94e62bc6ce7fc67328fb4 2024-12-03T05:17:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1733203115401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:35,403 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e3c6570475b6,37863,1733202887099 for 'test_row_2' in 9376883f3ccddf1028e50b0de6a8fad3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T05:17:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/fdfa642afaea46e49a532a2381a1263e as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e 2024-12-03T05:17:35,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e, entries=200, sequenceid=295, filesize=39.0 K 2024-12-03T05:17:35,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/5d98f91aba0f4666a83f41e30a590bc3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3 2024-12-03T05:17:35,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T05:17:35,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/4a12dca1eeb94e62bc6ce7fc67328fb4 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4 2024-12-03T05:17:35,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T05:17:35,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=46.96 KB/48090 for 9376883f3ccddf1028e50b0de6a8fad3 in 1360ms, sequenceid=295, compaction requested=true 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:35,425 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:35,425 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:35,426 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,426 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,426 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6a123bef03c84b8b9dbe5cd3b184e8fd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=36.2 K 2024-12-03T05:17:35,426 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=100.3 K 2024-12-03T05:17:35,426 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e] 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a123bef03c84b8b9dbe5cd3b184e8fd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0617e0ef6c1a44968a1df7c74d8eeb57, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5cfd38d46943b0be9676a3e8799d8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733203051702 2024-12-03T05:17:35,426 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae92d6eb01e46e2aceb49bc1d89c547, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733203051702 2024-12-03T05:17:35,427 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d98f91aba0f4666a83f41e30a590bc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:35,427 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdfa642afaea46e49a532a2381a1263e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:35,432 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:35,433 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#539 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:35,433 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831 is 50, key is test_row_0/B:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:35,437 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037b9369d6c100498786e003be4e0cad27_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:35,438 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037b9369d6c100498786e003be4e0cad27_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:35,439 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037b9369d6c100498786e003be4e0cad27_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:35,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742468_1644 (size=12915) 2024-12-03T05:17:35,446 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831 2024-12-03T05:17:35,451 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into c16c9ffdbb4a4b01a0a1ea7e6cb8f831(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:35,451 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:35,451 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=13, startTime=1733203055425; duration=0sec 2024-12-03T05:17:35,451 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:35,451 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:35,451 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:35,452 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:35,452 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:35,452 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,452 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17ae5ab1a80740e9951a99a143ef5ed3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=36.2 K 2024-12-03T05:17:35,452 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ae5ab1a80740e9951a99a143ef5ed3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733203050534 2024-12-03T05:17:35,452 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a572f40fae84ab784ad50ed37835205, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733203051702 2024-12-03T05:17:35,453 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a12dca1eeb94e62bc6ce7fc67328fb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:35,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742469_1645 (size=4469) 2024-12-03T05:17:35,454 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#540 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:35,455 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d95f7235b62c42f1a6f5d2514e45b5f2 is 175, key is test_row_0/A:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:35,466 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:35,466 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/7e41dc05bd5647d0b5d5430000aceb3f is 50, key is test_row_0/C:col10/1733203054063/Put/seqid=0 2024-12-03T05:17:35,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742470_1646 (size=31869) 2024-12-03T05:17:35,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742471_1647 (size=12915) 2024-12-03T05:17:35,479 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/7e41dc05bd5647d0b5d5430000aceb3f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7e41dc05bd5647d0b5d5430000aceb3f 2024-12-03T05:17:35,483 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into 7e41dc05bd5647d0b5d5430000aceb3f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:35,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:35,483 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=13, startTime=1733203055425; duration=0sec 2024-12-03T05:17:35,483 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:35,484 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:35,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T05:17:35,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:35,500 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T05:17:35,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:35,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c0a6a3883cac4bfe830e3f7b5e512d95_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203054090/Put/seqid=0 2024-12-03T05:17:35,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742472_1648 (size=12454) 2024-12-03T05:17:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:35,540 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c0a6a3883cac4bfe830e3f7b5e512d95_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c0a6a3883cac4bfe830e3f7b5e512d95_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:35,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/50877dd7adc74dc39e586be0a5474756, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/50877dd7adc74dc39e586be0a5474756 is 175, key is test_row_0/A:col10/1733203054090/Put/seqid=0 2024-12-03T05:17:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:35,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742473_1649 (size=31255) 2024-12-03T05:17:35,871 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d95f7235b62c42f1a6f5d2514e45b5f2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2 2024-12-03T05:17:35,874 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into d95f7235b62c42f1a6f5d2514e45b5f2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:35,874 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:35,874 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=13, startTime=1733203055425; duration=0sec 2024-12-03T05:17:35,875 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:35,875 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:35,957 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/50877dd7adc74dc39e586be0a5474756 2024-12-03T05:17:35,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a7461997d27d48369fb2d8ac64601ab2 is 50, key is test_row_0/B:col10/1733203054090/Put/seqid=0 2024-12-03T05:17:36,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742474_1650 (size=12301) 2024-12-03T05:17:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203056263, max timestamp=1733203056271Num uniques 859; 2024-12-03T05:17:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:36,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203056263, max timestamp=1733203056271Num uniques 859; 2024-12-03T05:17:36,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:36,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733203056263, max timestamp=1733203056271Num uniques 859; 2024-12-03T05:17:36,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T05:17:36,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203116305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203116306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203116308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,411 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a7461997d27d48369fb2d8ac64601ab2 2024-12-03T05:17:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/7b2835eb1be14bd398bbb8127b7b4738 is 50, key is test_row_0/C:col10/1733203054090/Put/seqid=0 2024-12-03T05:17:36,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203116420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203116420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203116420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742475_1651 (size=12301) 2024-12-03T05:17:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203116630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203116630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203116630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:36,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/7b2835eb1be14bd398bbb8127b7b4738 2024-12-03T05:17:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/50877dd7adc74dc39e586be0a5474756 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756 2024-12-03T05:17:36,833 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756, entries=150, sequenceid=307, filesize=30.5 K 2024-12-03T05:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/a7461997d27d48369fb2d8ac64601ab2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2 2024-12-03T05:17:36,836 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2, entries=150, sequenceid=307, filesize=12.0 K 2024-12-03T05:17:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/7b2835eb1be14bd398bbb8127b7b4738 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738 2024-12-03T05:17:36,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738, entries=150, sequenceid=307, filesize=12.0 K 2024-12-03T05:17:36,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 9376883f3ccddf1028e50b0de6a8fad3 in 1340ms, sequenceid=307, compaction requested=false 2024-12-03T05:17:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-03T05:17:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-03T05:17:36,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T05:17:36,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4190 sec 2024-12-03T05:17:36,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.4220 sec 2024-12-03T05:17:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203056298, max timestamp=1733203056942Num uniques 1018; 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203056298, max timestamp=1733203056941Num uniques 1018; 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733203056298, max timestamp=1733203056942Num uniques 1009; 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T05:17:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-03T05:17:36,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=40.25 KB heapSize=105.23 KB 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T05:17:36,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba72c44ca06c4b6ca041c47cff31110a_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:36,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742476_1652 (size=14994) 2024-12-03T05:17:36,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:36,972 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba72c44ca06c4b6ca041c47cff31110a_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba72c44ca06c4b6ca041c47cff31110a_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:36,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d48e490d31b24d30b09c9378cbc0a16b, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:36,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d48e490d31b24d30b09c9378cbc0a16b is 175, key is test_row_0/A:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203056965, max timestamp=1733203056972Num uniques 859; 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203056965, max timestamp=1733203056972Num uniques 859; 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733203056965, max timestamp=1733203056972Num uniques 859; 2024-12-03T05:17:36,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T05:17:36,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742477_1653 (size=39949) 2024-12-03T05:17:37,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733203056983, max timestamp=1733203057003Num uniques 1018; 2024-12-03T05:17:37,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T05:17:37,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733203056983, max timestamp=1733203057003Num uniques 1018; 2024-12-03T05:17:37,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T05:17:37,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733203056983, max timestamp=1733203057003Num uniques 1018; 2024-12-03T05:17:37,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3c6570475b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T05:17:37,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203117043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203117045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203117046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203117153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203117160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203117160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203117361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203117370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203117370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d48e490d31b24d30b09c9378cbc0a16b 2024-12-03T05:17:37,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/4a52af92675c46a291030f12917cfcef is 50, key is test_row_0/B:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:37,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742478_1654 (size=12301) 2024-12-03T05:17:37,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/4a52af92675c46a291030f12917cfcef 2024-12-03T05:17:37,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/e43a9ff7b0f94d77bcea6e382e20a21f is 50, key is test_row_0/C:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:37,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742479_1655 (size=12301) 2024-12-03T05:17:37,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/e43a9ff7b0f94d77bcea6e382e20a21f 2024-12-03T05:17:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d48e490d31b24d30b09c9378cbc0a16b as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b 2024-12-03T05:17:37,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b, entries=200, sequenceid=348, filesize=39.0 K 2024-12-03T05:17:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/4a52af92675c46a291030f12917cfcef as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef 2024-12-03T05:17:37,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef, entries=150, sequenceid=348, filesize=12.0 K 2024-12-03T05:17:37,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/e43a9ff7b0f94d77bcea6e382e20a21f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f 2024-12-03T05:17:37,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f, entries=150, sequenceid=348, filesize=12.0 K 2024-12-03T05:17:37,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for 9376883f3ccddf1028e50b0de6a8fad3 in 461ms, sequenceid=348, compaction requested=true 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:37,406 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:37,406 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9376883f3ccddf1028e50b0de6a8fad3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T05:17:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:37,407 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:37,407 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/B is initiating minor compaction (all files) 2024-12-03T05:17:37,407 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/B in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:37,407 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=36.6 K 2024-12-03T05:17:37,407 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting c16c9ffdbb4a4b01a0a1ea7e6cb8f831, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:37,407 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting a7461997d27d48369fb2d8ac64601ab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733203054084 2024-12-03T05:17:37,408 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a52af92675c46a291030f12917cfcef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733203056306 2024-12-03T05:17:37,408 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:37,408 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/A is initiating minor compaction (all files) 2024-12-03T05:17:37,408 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/A in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:37,408 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=100.7 K 2024-12-03T05:17:37,408 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:37,408 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. files: [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b] 2024-12-03T05:17:37,408 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95f7235b62c42f1a6f5d2514e45b5f2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:37,409 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50877dd7adc74dc39e586be0a5474756, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733203054084 2024-12-03T05:17:37,409 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48e490d31b24d30b09c9378cbc0a16b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733203056306 2024-12-03T05:17:37,415 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:37,416 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#B#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:37,417 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/bae4b4e58b8c485f945666373433d1fa is 50, key is test_row_0/B:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:37,421 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033a35c80128f0478e81a1ce1c0a2c60b1_9376883f3ccddf1028e50b0de6a8fad3 store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:37,423 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033a35c80128f0478e81a1ce1c0a2c60b1_9376883f3ccddf1028e50b0de6a8fad3, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:37,423 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033a35c80128f0478e81a1ce1c0a2c60b1_9376883f3ccddf1028e50b0de6a8fad3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742480_1656 (size=13017) 2024-12-03T05:17:37,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742481_1657 (size=4469) 2024-12-03T05:17:37,427 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#A#compaction#549 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:37,427 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d79fd9bcca93431280c1cc35dd9e4f7c is 175, key is test_row_0/A:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:37,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742482_1658 (size=31971) 2024-12-03T05:17:37,436 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/d79fd9bcca93431280c1cc35dd9e4f7c as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d79fd9bcca93431280c1cc35dd9e4f7c 2024-12-03T05:17:37,441 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/A of 9376883f3ccddf1028e50b0de6a8fad3 into d79fd9bcca93431280c1cc35dd9e4f7c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:37,441 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:37,441 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/A, priority=13, startTime=1733203057406; duration=0sec 2024-12-03T05:17:37,441 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T05:17:37,441 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:A 2024-12-03T05:17:37,441 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T05:17:37,442 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T05:17:37,442 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1541): 9376883f3ccddf1028e50b0de6a8fad3/C is initiating minor compaction (all files) 2024-12-03T05:17:37,442 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9376883f3ccddf1028e50b0de6a8fad3/C in TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:37,442 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7e41dc05bd5647d0b5d5430000aceb3f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f] into tmpdir=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp, totalSize=36.6 K 2024-12-03T05:17:37,442 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e41dc05bd5647d0b5d5430000aceb3f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733203052917 2024-12-03T05:17:37,442 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2835eb1be14bd398bbb8127b7b4738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733203054084 2024-12-03T05:17:37,442 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43a9ff7b0f94d77bcea6e382e20a21f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1733203056306 2024-12-03T05:17:37,448 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9376883f3ccddf1028e50b0de6a8fad3#C#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:17:37,449 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/6c2f0c2440244b558e0b540680993fc8 is 50, key is test_row_0/C:col10/1733203056941/Put/seqid=0 2024-12-03T05:17:37,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742483_1659 (size=13017) 2024-12-03T05:17:37,458 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/6c2f0c2440244b558e0b540680993fc8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6c2f0c2440244b558e0b540680993fc8 2024-12-03T05:17:37,463 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/C of 9376883f3ccddf1028e50b0de6a8fad3 into 6c2f0c2440244b558e0b540680993fc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:37,463 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:37,463 INFO [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/C, priority=13, startTime=1733203057406; duration=0sec 2024-12-03T05:17:37,463 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:37,463 DEBUG [RS:0;e3c6570475b6:37863-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:C 2024-12-03T05:17:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:37,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-12-03T05:17:37,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:17:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:17:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T05:17:37,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203117690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203117693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203117696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd34068754664412bdeec1fe23252100_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203057044/Put/seqid=0 2024-12-03T05:17:37,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742484_1660 (size=14994) 2024-12-03T05:17:37,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203117800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203117801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203117811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:37,834 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/bae4b4e58b8c485f945666373433d1fa as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/bae4b4e58b8c485f945666373433d1fa 2024-12-03T05:17:37,837 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9376883f3ccddf1028e50b0de6a8fad3/B of 9376883f3ccddf1028e50b0de6a8fad3 into bae4b4e58b8c485f945666373433d1fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:17:37,837 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:37,837 INFO [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., storeName=9376883f3ccddf1028e50b0de6a8fad3/B, priority=13, startTime=1733203057406; duration=0sec 2024-12-03T05:17:37,837 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T05:17:37,837 DEBUG [RS:0;e3c6570475b6:37863-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9376883f3ccddf1028e50b0de6a8fad3:B 2024-12-03T05:17:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203118010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203118010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203118021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:38,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd34068754664412bdeec1fe23252100_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd34068754664412bdeec1fe23252100_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:38,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1c19a3cdefdd491d9b3e43f60a7040f9, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:38,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1c19a3cdefdd491d9b3e43f60a7040f9 is 175, key is test_row_0/A:col10/1733203057044/Put/seqid=0 2024-12-03T05:17:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742485_1661 (size=39949) 2024-12-03T05:17:38,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203118320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203118321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203118335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1c19a3cdefdd491d9b3e43f60a7040f9 2024-12-03T05:17:38,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/428cd966b05f4ddb94023af0e19bd723 is 50, key is test_row_0/B:col10/1733203057044/Put/seqid=0 2024-12-03T05:17:38,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742486_1662 (size=12301) 2024-12-03T05:17:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T05:17:38,591 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:38,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T05:17:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-03T05:17:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:38,596 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T05:17:38,597 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T05:17:38,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T05:17:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:38,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:38,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:38,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:38,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:38,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59266 deadline: 1733203118830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59252 deadline: 1733203118830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T05:17:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59294 deadline: 1733203118851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 , the old value is region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T05:17:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., hostname=e3c6570475b6,37863,1733202887099, seqNum=5 because the exception is null or not the one we care about 2024-12-03T05:17:38,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:38,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:38,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/428cd966b05f4ddb94023af0e19bd723 2024-12-03T05:17:38,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/2f824e435e0d4994a0050fe980871fe5 is 50, key is test_row_0/C:col10/1733203057044/Put/seqid=0 2024-12-03T05:17:39,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742487_1663 (size=12301) 2024-12-03T05:17:39,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:39,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:39,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:39,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:17:39,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/2f824e435e0d4994a0050fe980871fe5 2024-12-03T05:17:39,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/1c19a3cdefdd491d9b3e43f60a7040f9 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1c19a3cdefdd491d9b3e43f60a7040f9 2024-12-03T05:17:39,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1c19a3cdefdd491d9b3e43f60a7040f9, entries=200, sequenceid=405, filesize=39.0 K 2024-12-03T05:17:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/428cd966b05f4ddb94023af0e19bd723 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/428cd966b05f4ddb94023af0e19bd723 2024-12-03T05:17:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/428cd966b05f4ddb94023af0e19bd723, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T05:17:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/2f824e435e0d4994a0050fe980871fe5 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/2f824e435e0d4994a0050fe980871fe5 2024-12-03T05:17:39,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/2f824e435e0d4994a0050fe980871fe5, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T05:17:39,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for 9376883f3ccddf1028e50b0de6a8fad3 in 1737ms, sequenceid=405, compaction requested=false 2024-12-03T05:17:39,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:39,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37863 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T05:17:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:39,517 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038677119d346c43308272dd818568c969_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203057691/Put/seqid=0 2024-12-03T05:17:39,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742488_1664 (size=12454) 2024-12-03T05:17:39,655 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-12-03T05:17:39,655 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,655 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,655 INFO [Thread-2540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2540. 2024-12-03T05:17:39,655 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,655 DEBUG [Thread-2540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,655 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,655 INFO [Thread-2548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2548. 2024-12-03T05:17:39,655 DEBUG [Thread-2548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,655 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,655 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,656 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,657 INFO [Thread-2546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2546. 2024-12-03T05:17:39,657 DEBUG [Thread-2546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,657 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,657 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,661 INFO [Thread-2544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2544. 2024-12-03T05:17:39,661 DEBUG [Thread-2544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,661 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,661 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:39,842 INFO [Thread-2529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2529. 2024-12-03T05:17:39,842 DEBUG [Thread-2529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,842 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,842 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,852 INFO [Thread-2535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2535. 2024-12-03T05:17:39,852 DEBUG [Thread-2535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,852 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,852 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37863 {}] regionserver.HRegion(8855): Flush requested on 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. as already flushing 2024-12-03T05:17:39,862 INFO [Thread-2533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2533. 2024-12-03T05:17:39,862 DEBUG [Thread-2533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:39,862 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,863 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:39,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:39,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038677119d346c43308272dd818568c969_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038677119d346c43308272dd818568c969_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/db502e1c35b94076bb4b7c40cffd857d, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/db502e1c35b94076bb4b7c40cffd857d is 175, key is test_row_0/A:col10/1733203057691/Put/seqid=0 2024-12-03T05:17:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742489_1665 (size=31255) 2024-12-03T05:17:40,367 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/db502e1c35b94076bb4b7c40cffd857d 2024-12-03T05:17:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/17e22f062b30477b99cad98e5487fdd2 is 50, key is test_row_0/B:col10/1733203057691/Put/seqid=0 2024-12-03T05:17:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742490_1666 (size=12301) 2024-12-03T05:17:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:40,777 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/17e22f062b30477b99cad98e5487fdd2 2024-12-03T05:17:40,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/274a60fefe8a40518cf289af15dd2726 is 50, key is test_row_0/C:col10/1733203057691/Put/seqid=0 2024-12-03T05:17:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742491_1667 (size=12301) 2024-12-03T05:17:41,185 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/274a60fefe8a40518cf289af15dd2726 2024-12-03T05:17:41,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/db502e1c35b94076bb4b7c40cffd857d as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/db502e1c35b94076bb4b7c40cffd857d 2024-12-03T05:17:41,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/db502e1c35b94076bb4b7c40cffd857d, entries=150, sequenceid=413, filesize=30.5 K 2024-12-03T05:17:41,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/17e22f062b30477b99cad98e5487fdd2 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/17e22f062b30477b99cad98e5487fdd2 2024-12-03T05:17:41,196 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/17e22f062b30477b99cad98e5487fdd2, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T05:17:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/274a60fefe8a40518cf289af15dd2726 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/274a60fefe8a40518cf289af15dd2726 2024-12-03T05:17:41,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/274a60fefe8a40518cf289af15dd2726, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T05:17:41,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=20.13 KB/20610 for 9376883f3ccddf1028e50b0de6a8fad3 in 1683ms, sequenceid=413, compaction requested=true 2024-12-03T05:17:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 9376883f3ccddf1028e50b0de6a8fad3: 2024-12-03T05:17:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-03T05:17:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-03T05:17:41,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-03T05:17:41,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6040 sec 2024-12-03T05:17:41,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.6110 sec 2024-12-03T05:17:42,262 INFO [Thread-2537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2537. 2024-12-03T05:17:42,262 DEBUG [Thread-2537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:42,262 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:42,262 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:42,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T05:17:42,751 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:45,442 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-12-03T05:17:45,442 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T05:17:45,442 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:45,442 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:45,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3173 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3239 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3248 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3170 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3224 2024-12-03T05:17:45,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T05:17:45,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:17:45,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:17:45,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:45,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:45,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:45,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T05:17:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:45,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203065445"}]},"ts":"1733203065445"} 2024-12-03T05:17:45,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T05:17:45,446 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T05:17:45,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T05:17:45,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, UNASSIGN}] 2024-12-03T05:17:45,448 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, UNASSIGN 2024-12-03T05:17:45,449 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=CLOSING, regionLocation=e3c6570475b6,37863,1733202887099 2024-12-03T05:17:45,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, UNASSIGN because future has completed 2024-12-03T05:17:45,450 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:17:45,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099}] 2024-12-03T05:17:45,483 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:17:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:45,603 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 9376883f3ccddf1028e50b0de6a8fad3, disabling compactions & flushes 2024-12-03T05:17:45,603 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. after waiting 0 ms 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:45,603 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 9376883f3ccddf1028e50b0de6a8fad3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=A 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=B 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9376883f3ccddf1028e50b0de6a8fad3, store=C 2024-12-03T05:17:45,603 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T05:17:45,607 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039dcf394ae42541e69432d5969d64ed8e_9376883f3ccddf1028e50b0de6a8fad3 is 50, key is test_row_0/A:col10/1733203059851/Put/seqid=0 2024-12-03T05:17:45,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742492_1668 (size=12454) 2024-12-03T05:17:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:46,011 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:17:46,014 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039dcf394ae42541e69432d5969d64ed8e_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039dcf394ae42541e69432d5969d64ed8e_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:46,015 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b2d2f1670b014568bc4acbe1b826f27f, store: [table=TestAcidGuarantees family=A region=9376883f3ccddf1028e50b0de6a8fad3] 2024-12-03T05:17:46,015 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b2d2f1670b014568bc4acbe1b826f27f is 175, key is test_row_0/A:col10/1733203059851/Put/seqid=0 2024-12-03T05:17:46,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742493_1669 (size=31255) 2024-12-03T05:17:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:46,419 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b2d2f1670b014568bc4acbe1b826f27f 2024-12-03T05:17:46,428 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/608012864aea4b7fb6c622894a9896ec is 50, key is test_row_0/B:col10/1733203059851/Put/seqid=0 2024-12-03T05:17:46,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742494_1670 (size=12301) 2024-12-03T05:17:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:46,838 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/608012864aea4b7fb6c622894a9896ec 2024-12-03T05:17:46,845 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5e1c6e300530414cb94c13e9d0749540 is 50, key is test_row_0/C:col10/1733203059851/Put/seqid=0 2024-12-03T05:17:46,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742495_1671 (size=12301) 2024-12-03T05:17:47,248 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5e1c6e300530414cb94c13e9d0749540 2024-12-03T05:17:47,251 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/A/b2d2f1670b014568bc4acbe1b826f27f as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b2d2f1670b014568bc4acbe1b826f27f 2024-12-03T05:17:47,254 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b2d2f1670b014568bc4acbe1b826f27f, entries=150, sequenceid=421, filesize=30.5 K 2024-12-03T05:17:47,255 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/B/608012864aea4b7fb6c622894a9896ec as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/608012864aea4b7fb6c622894a9896ec 2024-12-03T05:17:47,257 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/608012864aea4b7fb6c622894a9896ec, entries=150, sequenceid=421, filesize=12.0 K 2024-12-03T05:17:47,258 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/.tmp/C/5e1c6e300530414cb94c13e9d0749540 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5e1c6e300530414cb94c13e9d0749540 2024-12-03T05:17:47,260 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5e1c6e300530414cb94c13e9d0749540, entries=150, sequenceid=421, filesize=12.0 K 2024-12-03T05:17:47,261 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9376883f3ccddf1028e50b0de6a8fad3 in 1658ms, sequenceid=421, compaction requested=true 2024-12-03T05:17:47,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b] to archive 2024-12-03T05:17:47,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:47,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/89253c8258434ec388dbb3d9162beaf0 2024-12-03T05:17:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ac1475db1c5b4c14816b49bf0858cc6c 2024-12-03T05:17:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/67bf51ba82ea4958bda5ae4560730e84 2024-12-03T05:17:47,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/a00aa6cf93b34c75b275487813d57e9d 2024-12-03T05:17:47,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/3340490f165a4b6a8781e5a6e5711bac 2024-12-03T05:17:47,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bb4a675c11904a2bb843e233561bbfec 2024-12-03T05:17:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1d1898227bd54ed4b4394740aaabc570 2024-12-03T05:17:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/80b444afdeac4bf78e392e08fad838c6 2024-12-03T05:17:47,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/ed4a1f58910c430695cbf788f3906799 2024-12-03T05:17:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/5f0b6b7d76f74060824a58d19d7351a2 2024-12-03T05:17:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/43f64ea0f3b146ce8d25b13f466a3dc7 2024-12-03T05:17:47,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/bd7605bde1a548b485821daa8beafbc6 2024-12-03T05:17:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b63332aab9dc45adb2fb17c3329d295f 2024-12-03T05:17:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/c3811ea6098147ab84ca518d58011c2e 2024-12-03T05:17:47,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/45e8825a986e4e00a62995bcdbd19eba 2024-12-03T05:17:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/0617e0ef6c1a44968a1df7c74d8eeb57 2024-12-03T05:17:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/dae92d6eb01e46e2aceb49bc1d89c547 2024-12-03T05:17:47,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/fdfa642afaea46e49a532a2381a1263e 2024-12-03T05:17:47,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d95f7235b62c42f1a6f5d2514e45b5f2 2024-12-03T05:17:47,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/50877dd7adc74dc39e586be0a5474756 2024-12-03T05:17:47,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d48e490d31b24d30b09c9378cbc0a16b 2024-12-03T05:17:47,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/897f33e6b53841be83ba41ef3027a6e7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6f25e37634d2489eaded68f08d359d84, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/23cf691aaccb4a08ab1c91bb6880542b, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6a123bef03c84b8b9dbe5cd3b184e8fd, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef] to archive 2024-12-03T05:17:47,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:47,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f61d9af774004e53a4c8037a8f34b9b8 2024-12-03T05:17:47,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/238ef0b736064237b4ca36ccf1cc4cf9 2024-12-03T05:17:47,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/897f33e6b53841be83ba41ef3027a6e7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/897f33e6b53841be83ba41ef3027a6e7 2024-12-03T05:17:47,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5b8d3142f0254bbfba5dfdd0f968653e 2024-12-03T05:17:47,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a66ca028edb846fcac79dce2e42f5e9d 2024-12-03T05:17:47,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6f25e37634d2489eaded68f08d359d84 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6f25e37634d2489eaded68f08d359d84 2024-12-03T05:17:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/969f1e4579d94ce28e1bd6aceef1e9a3 2024-12-03T05:17:47,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/81604493ed954ce5935d272207a7a49a 2024-12-03T05:17:47,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/36666b4b6dec492c9cd22d2e506eaaab 2024-12-03T05:17:47,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/23cf691aaccb4a08ab1c91bb6880542b to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/23cf691aaccb4a08ab1c91bb6880542b 2024-12-03T05:17:47,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/0eead8776108475f91004e01c227ef3c 2024-12-03T05:17:47,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/ca5d0df8ef664221864188add0019345 2024-12-03T05:17:47,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/f62e4a6bfe15461b8d747994ee105805 2024-12-03T05:17:47,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6870471218924a1b8a342a2fb7a72d1a 2024-12-03T05:17:47,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6a123bef03c84b8b9dbe5cd3b184e8fd to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/6a123bef03c84b8b9dbe5cd3b184e8fd 2024-12-03T05:17:47,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/03dab5429d164bdfb556b5368eb8fcab 2024-12-03T05:17:47,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/cb5cfd38d46943b0be9676a3e8799d8a 2024-12-03T05:17:47,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/c16c9ffdbb4a4b01a0a1ea7e6cb8f831 2024-12-03T05:17:47,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/5d98f91aba0f4666a83f41e30a590bc3 2024-12-03T05:17:47,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/a7461997d27d48369fb2d8ac64601ab2 2024-12-03T05:17:47,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/4a52af92675c46a291030f12917cfcef 2024-12-03T05:17:47,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4218a28404fc44bbba9816ec32525894, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/db86d31a19e4436f916a8d135e7739a4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6b77765680024feeb070d1f76857fa48, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17ae5ab1a80740e9951a99a143ef5ed3, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7e41dc05bd5647d0b5d5430000aceb3f, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f] to archive 2024-12-03T05:17:47,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:17:47,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/72ea4663107f43229e6ff845f29a00d9 2024-12-03T05:17:47,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17a76be20add4d2d8bf62cc08beebc2f 2024-12-03T05:17:47,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4218a28404fc44bbba9816ec32525894 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4218a28404fc44bbba9816ec32525894 2024-12-03T05:17:47,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/0b5b8844e9854cb7aece6418e606ea0d 2024-12-03T05:17:47,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4dc7e6d50437447290da0c195f6805d3 2024-12-03T05:17:47,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/db86d31a19e4436f916a8d135e7739a4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/db86d31a19e4436f916a8d135e7739a4 2024-12-03T05:17:47,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/1fa6bc97faae45239113f13db53dc9aa 2024-12-03T05:17:47,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/92fa0b76fa7f438c91f17fb2d5bf7852 2024-12-03T05:17:47,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/342ffb3f967f47baad66ba857749f444 2024-12-03T05:17:47,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6b77765680024feeb070d1f76857fa48 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6b77765680024feeb070d1f76857fa48 2024-12-03T05:17:47,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/97444dccebcb4d9997646fb5230ccf28 2024-12-03T05:17:47,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a9afa71a994491f86ec5e00f38d9ea7 2024-12-03T05:17:47,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/33ad410f257544278f565af0123b7916 2024-12-03T05:17:47,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/3f30ffaf4d924d3baad8ea4cbff8f5ac 2024-12-03T05:17:47,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17ae5ab1a80740e9951a99a143ef5ed3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/17ae5ab1a80740e9951a99a143ef5ed3 2024-12-03T05:17:47,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/bfed0f86ed4e47229b0dc48020f6efad 2024-12-03T05:17:47,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5a572f40fae84ab784ad50ed37835205 2024-12-03T05:17:47,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7e41dc05bd5647d0b5d5430000aceb3f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7e41dc05bd5647d0b5d5430000aceb3f 2024-12-03T05:17:47,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/4a12dca1eeb94e62bc6ce7fc67328fb4 2024-12-03T05:17:47,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/7b2835eb1be14bd398bbb8127b7b4738 2024-12-03T05:17:47,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/e43a9ff7b0f94d77bcea6e382e20a21f 2024-12-03T05:17:47,321 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-12-03T05:17:47,321 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. 2024-12-03T05:17:47,321 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 9376883f3ccddf1028e50b0de6a8fad3: Waiting for close lock at 1733203065603Running coprocessor pre-close hooks at 1733203065603Disabling compacts and flushes for region at 1733203065603Disabling writes for close at 1733203065603Obtaining lock to block concurrent updates at 1733203065603Preparing flush snapshotting stores in 9376883f3ccddf1028e50b0de6a8fad3 at 1733203065603Finished memstore snapshotting TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733203065603Flushing stores of TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3. at 1733203065604 (+1 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/A: creating writer at 1733203065604Flushing mob file 9376883f3ccddf1028e50b0de6a8fad3/A: appending metadata at 1733203065607 (+3 ms)Flushing mob file 9376883f3ccddf1028e50b0de6a8fad3/A: closing flushed file at 1733203065607Flushing 9376883f3ccddf1028e50b0de6a8fad3/A: appending metadata at 1733203066015 (+408 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/A: closing flushed file at 1733203066015Flushing 9376883f3ccddf1028e50b0de6a8fad3/B: creating writer at 1733203066424 (+409 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/B: appending metadata at 1733203066427 (+3 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/B: closing flushed file at 1733203066427Flushing 9376883f3ccddf1028e50b0de6a8fad3/C: creating writer at 1733203066841 (+414 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/C: appending metadata at 1733203066844 (+3 ms)Flushing 9376883f3ccddf1028e50b0de6a8fad3/C: closing flushed file at 1733203066844Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@418f82bf: reopening flushed file at 1733203067251 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c83596f: reopening flushed file at 1733203067254 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce53d12: reopening flushed file at 1733203067257 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9376883f3ccddf1028e50b0de6a8fad3 in 1658ms, sequenceid=421, compaction requested=true at 1733203067261 (+4 ms)Writing region close event to WAL at 1733203067318 (+57 ms)Running coprocessor post-close hooks at 1733203067321 (+3 ms)Closed at 1733203067321 2024-12-03T05:17:47,323 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=9376883f3ccddf1028e50b0de6a8fad3, regionState=CLOSED 2024-12-03T05:17:47,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 because future has completed 2024-12-03T05:17:47,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-03T05:17:47,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 9376883f3ccddf1028e50b0de6a8fad3, server=e3c6570475b6,37863,1733202887099 in 1.8750 sec 2024-12-03T05:17:47,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-03T05:17:47,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9376883f3ccddf1028e50b0de6a8fad3, UNASSIGN in 1.8780 sec 2024-12-03T05:17:47,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-03T05:17:47,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8810 sec 2024-12-03T05:17:47,331 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203067331"}]},"ts":"1733203067331"} 2024-12-03T05:17:47,332 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T05:17:47,332 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T05:17:47,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8900 sec 2024-12-03T05:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T05:17:47,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:47,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T05:17:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T05:17:47,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,597 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C, FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits] 2024-12-03T05:17:47,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1c19a3cdefdd491d9b3e43f60a7040f9 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/1c19a3cdefdd491d9b3e43f60a7040f9 2024-12-03T05:17:47,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b2d2f1670b014568bc4acbe1b826f27f to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/b2d2f1670b014568bc4acbe1b826f27f 2024-12-03T05:17:47,608 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d79fd9bcca93431280c1cc35dd9e4f7c to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/d79fd9bcca93431280c1cc35dd9e4f7c 2024-12-03T05:17:47,610 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/db502e1c35b94076bb4b7c40cffd857d to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/A/db502e1c35b94076bb4b7c40cffd857d 2024-12-03T05:17:47,612 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/17e22f062b30477b99cad98e5487fdd2 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/17e22f062b30477b99cad98e5487fdd2 2024-12-03T05:17:47,614 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/428cd966b05f4ddb94023af0e19bd723 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/428cd966b05f4ddb94023af0e19bd723 2024-12-03T05:17:47,615 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/608012864aea4b7fb6c622894a9896ec to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/608012864aea4b7fb6c622894a9896ec 2024-12-03T05:17:47,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/bae4b4e58b8c485f945666373433d1fa to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/B/bae4b4e58b8c485f945666373433d1fa 2024-12-03T05:17:47,618 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/274a60fefe8a40518cf289af15dd2726 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/274a60fefe8a40518cf289af15dd2726 2024-12-03T05:17:47,620 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/2f824e435e0d4994a0050fe980871fe5 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/2f824e435e0d4994a0050fe980871fe5 2024-12-03T05:17:47,621 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5e1c6e300530414cb94c13e9d0749540 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/5e1c6e300530414cb94c13e9d0749540 2024-12-03T05:17:47,622 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6c2f0c2440244b558e0b540680993fc8 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/C/6c2f0c2440244b558e0b540680993fc8 2024-12-03T05:17:47,625 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits/424.seqid to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3/recovered.edits/424.seqid 2024-12-03T05:17:47,625 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/default/TestAcidGuarantees/9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T05:17:47,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:17:47,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T05:17:47,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f61b5cab0f94387ab49800e049931f6_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f61b5cab0f94387ab49800e049931f6_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,631 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033fe273ccc564499cbd89ae45579c93f1_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033fe273ccc564499cbd89ae45579c93f1_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035593d48d1ed74ff1847477a014910f91_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035593d48d1ed74ff1847477a014910f91_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120362f1faffaa6543a696ccd5d345d1ea43_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120362f1faffaa6543a696ccd5d345d1ea43_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203694dd082edf54923b93821130598d760_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203694dd082edf54923b93821130598d760_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c0564d582034439ba8dd0cf00390a8c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c0564d582034439ba8dd0cf00390a8c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f459f9db27c461fb7119e156ac0d077_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f459f9db27c461fb7119e156ac0d077_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,637 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038672ff3140a44651a2ea4de1f25e442c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038672ff3140a44651a2ea4de1f25e442c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038677119d346c43308272dd818568c969_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038677119d346c43308272dd818568c969_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,639 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039dcf394ae42541e69432d5969d64ed8e_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039dcf394ae42541e69432d5969d64ed8e_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa409b30b2d846a6b1d88f4c4c94da3c_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa409b30b2d846a6b1d88f4c4c94da3c_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,641 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba72c44ca06c4b6ca041c47cff31110a_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba72c44ca06c4b6ca041c47cff31110a_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,642 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bfae11a9bec049f09c0f05b9b754d8ed_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bfae11a9bec049f09c0f05b9b754d8ed_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,643 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c0a6a3883cac4bfe830e3f7b5e512d95_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c0a6a3883cac4bfe830e3f7b5e512d95_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,645 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2358af460d24511982ff591bad8158e_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c2358af460d24511982ff591bad8158e_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,646 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c641017930a24e9fba9d2543fe627279_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c641017930a24e9fba9d2543fe627279_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,647 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd34068754664412bdeec1fe23252100_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd34068754664412bdeec1fe23252100_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,648 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eca8599d05744ef6ad201d3cb1e30d1a_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eca8599d05744ef6ad201d3cb1e30d1a_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,649 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f77ce449c1624cbbb2d82b57eac857c3_9376883f3ccddf1028e50b0de6a8fad3 to hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f77ce449c1624cbbb2d82b57eac857c3_9376883f3ccddf1028e50b0de6a8fad3 2024-12-03T05:17:47,649 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T05:17:47,651 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,653 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T05:17:47,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T05:17:47,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T05:17:47,656 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733203067655"}]},"ts":"9223372036854775807"} 2024-12-03T05:17:47,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T05:17:47,657 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9376883f3ccddf1028e50b0de6a8fad3, NAME => 'TestAcidGuarantees,,1733203037441.9376883f3ccddf1028e50b0de6a8fad3.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T05:17:47,657 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T05:17:47,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733203067657"}]},"ts":"9223372036854775807"} 2024-12-03T05:17:47,658 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T05:17:47,659 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T05:17:47,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-12-03T05:17:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T05:17:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T05:17:47,701 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T05:17:47,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=443 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=761 (was 773), ProcessCount=11 (was 11), AvailableMemoryMB=1691 (was 2426) 2024-12-03T05:17:47,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T05:17:47,712 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:17:47,712 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:17:47,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:47,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:47,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:17:47,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T05:17:47,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-12-03T05:17:47,713 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e3c6570475b6,44191,1733202886389 2024-12-03T05:17:47,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T05:17:47,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:17:47,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T05:17:47,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T05:17:47,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:17:47,761 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:17:47,762 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:17:47,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:47,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:17:47,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:17:47,762 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e3c6570475b6,37863,1733202887099' ***** 2024-12-03T05:17:47,763 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T05:17:47,763 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T05:17:47,764 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(959): stopping server e3c6570475b6,37863,1733202887099 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e3c6570475b6:37863. 2024-12-03T05:17:47,764 DEBUG [RS:0;e3c6570475b6:37863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:17:47,764 DEBUG [RS:0;e3c6570475b6:37863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T05:17:47,764 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T05:17:47,765 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T05:17:47,765 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T05:17:47,765 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T05:17:47,766 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T05:17:47,766 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:47,766 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T05:17:47,766 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T05:17:47,766 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T05:17:47,766 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T05:17:47,766 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T05:17:47,767 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T05:17:47,787 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/info/2db74dfc7b7c4561970d8c100d82217a is 95, key is TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21./info:/1733202919131/DeleteFamily/seqid=0 2024-12-03T05:17:47,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742496_1672 (size=6514) 2024-12-03T05:17:47,804 INFO [regionserver/e3c6570475b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:17:47,966 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:48,131 INFO [regionserver/e3c6570475b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T05:17:48,131 INFO [regionserver/e3c6570475b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T05:17:48,167 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:48,191 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/info/2db74dfc7b7c4561970d8c100d82217a 2024-12-03T05:17:48,216 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/ns/4f8854415bb4404cab4307a429fb5593 is 93, key is TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21./ns:/1733202919086/DeleteFamily/seqid=0 2024-12-03T05:17:48,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742497_1673 (size=5906) 2024-12-03T05:17:48,367 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:48,567 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:48,620 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/ns/4f8854415bb4404cab4307a429fb5593 2024-12-03T05:17:48,635 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/rep_barrier/ca3046e1921f480c9b21795ad56b4357 is 102, key is TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21./rep_barrier:/1733202919086/DeleteFamily/seqid=0 2024-12-03T05:17:48,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742498_1674 (size=6025) 2024-12-03T05:17:48,767 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:48,968 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:49,038 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/rep_barrier/ca3046e1921f480c9b21795ad56b4357 2024-12-03T05:17:49,053 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/table/6656452ee8b9437e9caffe009993b8c8 is 96, key is TestAcidGuarantees,,1733202891295.fc274659350fd9dda12032ec9405be21./table:/1733202919086/DeleteFamily/seqid=0 2024-12-03T05:17:49,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742499_1675 (size=5936) 2024-12-03T05:17:49,168 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:49,368 DEBUG [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T05:17:49,456 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/table/6656452ee8b9437e9caffe009993b8c8 2024-12-03T05:17:49,461 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/info/2db74dfc7b7c4561970d8c100d82217a as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/info/2db74dfc7b7c4561970d8c100d82217a 2024-12-03T05:17:49,464 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/info/2db74dfc7b7c4561970d8c100d82217a, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T05:17:49,465 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/ns/4f8854415bb4404cab4307a429fb5593 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/ns/4f8854415bb4404cab4307a429fb5593 2024-12-03T05:17:49,469 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/ns/4f8854415bb4404cab4307a429fb5593, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T05:17:49,469 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/rep_barrier/ca3046e1921f480c9b21795ad56b4357 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/rep_barrier/ca3046e1921f480c9b21795ad56b4357 2024-12-03T05:17:49,471 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/rep_barrier/ca3046e1921f480c9b21795ad56b4357, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T05:17:49,472 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/.tmp/table/6656452ee8b9437e9caffe009993b8c8 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/table/6656452ee8b9437e9caffe009993b8c8 2024-12-03T05:17:49,473 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/table/6656452ee8b9437e9caffe009993b8c8, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T05:17:49,474 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false 2024-12-03T05:17:49,478 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T05:17:49,478 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T05:17:49,478 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T05:17:49,478 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733203067766Running coprocessor pre-close hooks at 1733203067766Disabling compacts and flushes for region at 1733203067766Disabling writes for close at 1733203067766Obtaining lock to block concurrent updates at 1733203067767 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733203067767Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733203067768 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733203067768Flushing 1588230740/info: creating writer at 1733203067769 (+1 ms)Flushing 1588230740/info: appending metadata at 1733203067786 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733203067786Flushing 1588230740/ns: creating writer at 1733203068200 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733203068216 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733203068216Flushing 1588230740/rep_barrier: creating writer at 1733203068623 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733203068635 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733203068635Flushing 1588230740/table: creating writer at 1733203069041 (+406 ms)Flushing 1588230740/table: appending metadata at 1733203069053 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733203069053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bb034a6: reopening flushed file at 1733203069460 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@600791de: reopening flushed file at 1733203069465 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5edcd1: reopening flushed file at 1733203069469 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9ad8eb: reopening flushed file at 1733203069471 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false at 1733203069474 (+3 ms)Writing region close event to WAL at 1733203069475 (+1 ms)Running coprocessor post-close hooks at 1733203069478 (+3 ms)Closed at 1733203069478 2024-12-03T05:17:49,479 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T05:17:49,569 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(976): stopping server e3c6570475b6,37863,1733202887099; all regions closed. 2024-12-03T05:17:49,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741834_1010 (size=25344) 2024-12-03T05:17:49,575 DEBUG [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/oldWALs 2024-12-03T05:17:49,575 INFO [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3c6570475b6%2C37863%2C1733202887099.meta:.meta(num 1733202890750) 2024-12-03T05:17:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741833_1009 (size=22176755) 2024-12-03T05:17:49,579 DEBUG [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/oldWALs 2024-12-03T05:17:49,579 INFO [RS:0;e3c6570475b6:37863 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3c6570475b6%2C37863%2C1733202887099:(num 1733202890245) 2024-12-03T05:17:49,579 DEBUG [RS:0;e3c6570475b6:37863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:17:49,579 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:17:49,579 INFO [RS:0;e3c6570475b6:37863 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:17:49,579 INFO [RS:0;e3c6570475b6:37863 {}] hbase.ChoreService(370): Chore service for: regionserver/e3c6570475b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T05:17:49,579 INFO [RS:0;e3c6570475b6:37863 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:17:49,579 INFO [regionserver/e3c6570475b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:17:49,580 INFO [RS:0;e3c6570475b6:37863 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37863 2024-12-03T05:17:49,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:17:49,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e3c6570475b6,37863,1733202887099 2024-12-03T05:17:49,590 INFO [RS:0;e3c6570475b6:37863 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:17:49,600 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e3c6570475b6,37863,1733202887099] 2024-12-03T05:17:49,610 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e3c6570475b6,37863,1733202887099 already deleted, retry=false 2024-12-03T05:17:49,611 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e3c6570475b6,37863,1733202887099 expired; onlineServers=0 2024-12-03T05:17:49,611 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e3c6570475b6,44191,1733202886389' ***** 2024-12-03T05:17:49,611 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T05:17:49,611 INFO [M:0;e3c6570475b6:44191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:17:49,611 INFO [M:0;e3c6570475b6:44191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:17:49,611 DEBUG [M:0;e3c6570475b6:44191 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T05:17:49,611 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T05:17:49,611 DEBUG [M:0;e3c6570475b6:44191 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T05:17:49,611 DEBUG [master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733202889953 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733202889953,5,FailOnTimeoutGroup] 2024-12-03T05:17:49,611 DEBUG [master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733202889949 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733202889949,5,FailOnTimeoutGroup] 2024-12-03T05:17:49,611 INFO [M:0;e3c6570475b6:44191 {}] hbase.ChoreService(370): Chore service for: master/e3c6570475b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T05:17:49,612 INFO [M:0;e3c6570475b6:44191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:17:49,612 DEBUG [M:0;e3c6570475b6:44191 {}] master.HMaster(1795): Stopping service threads 2024-12-03T05:17:49,612 INFO [M:0;e3c6570475b6:44191 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T05:17:49,612 INFO [M:0;e3c6570475b6:44191 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T05:17:49,612 ERROR [M:0;e3c6570475b6:44191 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T05:17:49,612 INFO [M:0;e3c6570475b6:44191 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T05:17:49,613 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T05:17:49,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T05:17:49,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:17:49,621 DEBUG [M:0;e3c6570475b6:44191 {}] zookeeper.ZKUtil(347): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T05:17:49,621 WARN [M:0;e3c6570475b6:44191 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T05:17:49,622 INFO [M:0;e3c6570475b6:44191 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/.lastflushedseqids 2024-12-03T05:17:49,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742500_1676 (size=72) 2024-12-03T05:17:49,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:17:49,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37863-0x101999d37940001, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:17:49,701 INFO [RS:0;e3c6570475b6:37863 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:17:49,701 INFO [RS:0;e3c6570475b6:37863 {}] regionserver.HRegionServer(1031): Exiting; stopping=e3c6570475b6,37863,1733202887099; zookeeper connection closed. 2024-12-03T05:17:49,701 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b48d4aa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b48d4aa 2024-12-03T05:17:49,702 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T05:17:50,034 INFO [M:0;e3c6570475b6:44191 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T05:17:50,034 INFO [M:0;e3c6570475b6:44191 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T05:17:50,034 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T05:17:50,035 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:17:50,035 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:17:50,035 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T05:17:50,035 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:17:50,035 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=794.84 KB heapSize=975.05 KB 2024-12-03T05:17:50,051 DEBUG [M:0;e3c6570475b6:44191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd5c10f289664fe09036a4c7e84d1e14 is 82, key is hbase:meta,,1/info:regioninfo/1733202890826/Put/seqid=0 2024-12-03T05:17:50,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742501_1677 (size=5672) 2024-12-03T05:17:50,454 INFO [M:0;e3c6570475b6:44191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2234 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd5c10f289664fe09036a4c7e84d1e14 2024-12-03T05:17:50,483 DEBUG [M:0;e3c6570475b6:44191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb2d10be84ebaa741369a2bb55bb3 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1733202921950/Put/seqid=0 2024-12-03T05:17:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742502_1678 (size=45077) 2024-12-03T05:17:50,890 INFO [M:0;e3c6570475b6:44191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=794.28 KB at sequenceid=2234 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb2d10be84ebaa741369a2bb55bb3 2024-12-03T05:17:50,893 INFO [M:0;e3c6570475b6:44191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7bfb2d10be84ebaa741369a2bb55bb3 2024-12-03T05:17:50,906 DEBUG [M:0;e3c6570475b6:44191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7878243a0a514760b702ae827d4175bd is 69, key is e3c6570475b6,37863,1733202887099/rs:state/1733202889998/Put/seqid=0 2024-12-03T05:17:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073742503_1679 (size=5156) 2024-12-03T05:17:51,312 INFO [M:0;e3c6570475b6:44191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2234 (bloomFilter=true), to=hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7878243a0a514760b702ae827d4175bd 2024-12-03T05:17:51,316 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd5c10f289664fe09036a4c7e84d1e14 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd5c10f289664fe09036a4c7e84d1e14 2024-12-03T05:17:51,318 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd5c10f289664fe09036a4c7e84d1e14, entries=8, sequenceid=2234, filesize=5.5 K 2024-12-03T05:17:51,319 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb2d10be84ebaa741369a2bb55bb3 as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7bfb2d10be84ebaa741369a2bb55bb3 2024-12-03T05:17:51,321 INFO [M:0;e3c6570475b6:44191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7bfb2d10be84ebaa741369a2bb55bb3 2024-12-03T05:17:51,321 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7bfb2d10be84ebaa741369a2bb55bb3, entries=168, sequenceid=2234, filesize=44.0 K 2024-12-03T05:17:51,322 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7878243a0a514760b702ae827d4175bd as hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7878243a0a514760b702ae827d4175bd 2024-12-03T05:17:51,324 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46669/user/jenkins/test-data/545084cb-3f93-eb71-ac13-1aae71afcfdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7878243a0a514760b702ae827d4175bd, entries=1, sequenceid=2234, filesize=5.0 K 2024-12-03T05:17:51,325 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(3140): Finished flush of dataSize ~794.84 KB/813915, heapSize ~974.75 KB/998144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2234, compaction requested=false 2024-12-03T05:17:51,327 INFO [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:17:51,327 DEBUG [M:0;e3c6570475b6:44191 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733203070034Disabling compacts and flushes for region at 1733203070034Disabling writes for close at 1733203070035 (+1 ms)Obtaining lock to block concurrent updates at 1733203070035Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733203070035Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=813915, getHeapSize=998384, getOffHeapSize=0, getCellsCount=2427 at 1733203070036 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733203070037 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733203070037Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733203070051 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733203070051Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733203070458 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733203070482 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733203070482Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733203070894 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733203070906 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733203070906Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4200ba80: reopening flushed file at 1733203071315 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7323873f: reopening flushed file at 1733203071318 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2584aa15: reopening flushed file at 1733203071321 (+3 ms)Finished flush of dataSize ~794.84 KB/813915, heapSize ~974.75 KB/998144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2234, compaction requested=false at 1733203071325 (+4 ms)Writing region close event to WAL at 1733203071326 (+1 ms)Closed at 1733203071327 (+1 ms) 2024-12-03T05:17:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42545 is added to blk_1073741830_1006 (size=960316) 2024-12-03T05:17:51,329 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:17:51,330 INFO [M:0;e3c6570475b6:44191 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T05:17:51,330 INFO [M:0;e3c6570475b6:44191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44191 2024-12-03T05:17:51,330 INFO [M:0;e3c6570475b6:44191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:17:51,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:17:51,521 INFO [M:0;e3c6570475b6:44191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:17:51,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44191-0x101999d37940000, quorum=127.0.0.1:50539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:17:51,526 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:17:51,528 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:17:51,528 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:17:51,528 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:17:51,528 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.log.dir/,STOPPED} 2024-12-03T05:17:51,531 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T05:17:51,531 WARN [BP-738945746-172.17.0.3-1733202882469 heartbeating to localhost/127.0.0.1:46669 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T05:17:51,531 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T05:17:51,531 WARN [BP-738945746-172.17.0.3-1733202882469 heartbeating to localhost/127.0.0.1:46669 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-738945746-172.17.0.3-1733202882469 (Datanode Uuid 7ded0889-f37b-42aa-ac59-1df2d1988e00) service to localhost/127.0.0.1:46669 2024-12-03T05:17:51,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data1/current/BP-738945746-172.17.0.3-1733202882469 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:17:51,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/cluster_882e4952-c5a1-f2a8-df02-2d91bc05ad87/data/data2/current/BP-738945746-172.17.0.3-1733202882469 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:17:51,534 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T05:17:51,540 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T05:17:51,540 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:17:51,541 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:17:51,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:17:51,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25f0728c-6ba4-e46b-ceff-01e1313f9f3d/hadoop.log.dir/,STOPPED} 2024-12-03T05:17:51,559 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T05:17:51,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down